diff --git a/configure b/configure index d2ffd6b00bd..87e8756033f 100755 --- a/configure +++ b/configure @@ -721,6 +721,8 @@ with_gp_stats_collector with_diskquota with_libbz2 with_zstd +LZO_LIBS +with_lzo LZ4_LIBS LZ4_CFLAGS with_lz4 @@ -944,6 +946,7 @@ with_zlib with_lz4 with_zstd with_libbz2 +with_lzo with_diskquota with_gp_stats_collector with_rt @@ -1697,6 +1700,7 @@ Optional Packages: --with-zstd build with ZSTD support --without-libbz2 do not use bzip2 --with-zstd do not build with Zstandard + --with-lzo build with LZO support --with-diskquota build with diskquota extension --with-gp-stats-collector build with stats collector extension @@ -11574,6 +11578,115 @@ fi $as_echo "$with_zstd" >&6; } +# +# LZO +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZO support" >&5 +$as_echo_n "checking whether to build with LZO support... " >&6; } + + + +# Check whether --with-lzo was given. +if test "${with_lzo+set}" = set; then : + withval=$with_lzo; + case $withval in + yes) + +$as_echo "#define USE_LZO 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lzo option" "$LINENO" 5 + ;; + esac + +else + with_lzo=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lzo" >&5 +$as_echo "$with_lzo" >&6; } + + +if test "$with_lzo" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking lzo/lzo1x.h usability" >&5 +$as_echo_n "checking lzo/lzo1x.h usability... " >&6; } +if ${ac_cv_header_lzo_lzo1x_h+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default +#include +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_lzo_lzo1x_h=yes +else + ac_cv_header_lzo_lzo1x_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_lzo_lzo1x_h" >&5 +$as_echo "$ac_cv_header_lzo_lzo1x_h" >&6; } + + +if test "x$ac_cv_header_lzo_lzo1x_h" = x""yes; then : + : +else + as_fn_error $? "lzo/lzo1x.h header not found. Please install liblzo2-dev." "$LINENO" 5 +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for lzo1x_decompress_safe in -llzo2" >&5 +$as_echo_n "checking for lzo1x_decompress_safe in -llzo2... " >&6; } +if ${ac_cv_lib_lzo2_lzo1x_decompress_safe+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-llzo2 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Suppress warnings for systems that declare it in a header. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char lzo1x_decompress_safe (); +int +main () +{ +return lzo1x_decompress_safe (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_lzo2_lzo1x_decompress_safe=yes +else + ac_cv_lib_lzo2_lzo1x_decompress_safe=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzo2_lzo1x_decompress_safe" >&5 +$as_echo "$ac_cv_lib_lzo2_lzo1x_decompress_safe" >&6; } +if test "x$ac_cv_lib_lzo2_lzo1x_decompress_safe" = x""yes; then : + LZO_LIBS="-llzo2" +else + as_fn_error $? "liblzo2 library not found. Please install liblzo2-dev." "$LINENO" 5 +fi + +fi + + # # diskquota # diff --git a/configure.ac b/configure.ac index dd259c4f07f..571af089f32 100644 --- a/configure.ac +++ b/configure.ac @@ -1466,6 +1466,23 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# LZO +# +AC_MSG_CHECKING([whether to build with LZO support]) +AH_TEMPLATE([USE_LZO], [Define to 1 to build with LZO support. (--with-lzo)]) +PGAC_ARG_BOOL(with, lzo, no, [build with LZO support], + [AC_DEFINE([USE_LZO], 1)]) +AC_MSG_RESULT([$with_lzo]) +AC_SUBST(with_lzo) + +if test "$with_lzo" = yes; then + AC_CHECK_HEADER([lzo/lzo1x.h], [], [AC_MSG_ERROR([lzo/lzo1x.h header not found. Please install liblzo2-dev.])]) + AC_CHECK_LIB([lzo2], [lzo1x_decompress_safe], [LZO_LIBS="-llzo2"], + [AC_MSG_ERROR([liblzo2 library not found. Please install liblzo2-dev.])]) + AC_SUBST(LZO_LIBS) +fi + # # diskquota # diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile index 3ab40229020..fc2720a2c08 100644 --- a/devops/deploy/docker/build/rocky10/Dockerfile +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -122,6 +122,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index eed55e4b171..a1f6eacb4e9 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -104,6 +104,7 @@ RUN dnf makecache && \ libuuid-devel \ libxml2-devel \ libzstd-devel \ + lzo-devel \ lsof \ lz4 \ lz4-devel \ diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 592d9b2e10a..17ff9601efc 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -124,6 +124,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 index 215c32f452d..7750a79b240 100644 --- a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 +++ b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 @@ -81,6 +81,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/sandbox/Dockerfile.main.rockylinux9 b/devops/sandbox/Dockerfile.main.rockylinux9 index 03619cda4ba..676e230e6ca 100644 --- a/devops/sandbox/Dockerfile.main.rockylinux9 +++ b/devops/sandbox/Dockerfile.main.rockylinux9 @@ -86,6 +86,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 74ba8d0c370..bd47d02c289 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -268,6 +268,8 @@ YAML_LIBS = @YAML_LIBS@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ +with_lzo = @with_lzo@ +LZO_LIBS = @LZO_LIBS@ EVENT_LIBS = @EVENT_LIBS@ with_diskquota = @with_diskquota@ with_gp_stats_collector = @with_gp_stats_collector@ @@ -335,8 +337,8 @@ LDAP_LIBS_BE = @LDAP_LIBS_BE@ UUID_LIBS = @UUID_LIBS@ LLVM_LIBS=@LLVM_LIBS@ -# Cloudberry uses zstd in frontend and backend -LIBS := $(LIBS) $(ZSTD_LIBS) +# Cloudberry uses zstd and lzo in frontend and backend +LIBS := $(LIBS) $(ZSTD_LIBS) $(LZO_LIBS) # It's critical that within LDFLAGS, all -L switches pointing to build-tree # directories come before any -L switches pointing to external directories. diff --git a/src/backend/utils/misc/fstream/gfile.c b/src/backend/utils/misc/fstream/gfile.c index 070ca1c649c..b3c9d755fc5 100644 --- a/src/backend/utils/misc/fstream/gfile.c +++ b/src/backend/utils/misc/fstream/gfile.c @@ -253,7 +253,605 @@ static int bz_file_open(gfile_t *fd) fd->u.bz->s.next_in = fd->u.bz->in; fd->read = bz_file_read; fd->close = bz_file_close; - + + return 0; +} +#endif + +#ifdef USE_LZO +/* + * LZO-compressed file support (standard lzop container format). + * + * Uses in-process liblzo2 decompression, following the same pattern as + * .gz (zlib) / .bz2 (bzlib) / .zst (zstd). The file format is auto-detected + * in lzo_file_open() by probing the first 9 bytes: + * + * - standard lzop container format: 9-byte magic + full header + block + * checksums + * - Hadoop Raw LZO format: no magic/header/checksum, plain LZO block + * stream (handled as a bonus path; only the standard lzop format is + * advertised via the .lzo extension) + * + * lzop header layout: + * magic 9 bytes fixed magic \x89LZO\x00\x0d\x0a\x1a\x0a + * version 2 bytes version (big endian) + * lib_version 2 bytes library version + * ver_needed 2 bytes minimum version required to decompress + * method 1 byte compression algorithm + * level 1 byte compression level + * flags 4 bytes flags (control checksum types etc.) + * mode 4 bytes file mode + * mtime_low 4 bytes mtime low 32 bits + * mtime_high 4 bytes mtime high 32 bits + * [extra_ver] 1 byte if F_H_EXTRA_FIELD(0x40) set + * [filter] 4 bytes if F_H_FILTER(0x800) set + * name_len 1 byte original file name length + * name N bytes original file name + * [path_len] 4 bytes if F_H_PATH(0x2000) set + * [path] N bytes if F_H_PATH(0x2000) set + * checksum 4 bytes header checksum + * + * Data block layout (common to both formats): + * uncomp_len 4 bytes decompressed size, big endian (0 = EOF marker) + * comp_len 4 bytes compressed size, big endian + * [d_adler32] 4 bytes adler32 of decompressed data (F_ADLER32_D=0x01) + * [d_crc32] 4 bytes crc32 of decompressed data (F_CRC32_D=0x100) + * [c_adler32] 4 bytes adler32 of compressed data (F_ADLER32_C=0x02) + * [c_crc32] 4 bytes crc32 of compressed data (F_CRC32_C=0x200) + * data comp_len bytes LZO compressed data (or raw data if + * incompressible) + * + * The only difference between the two formats is whether the magic/header + * and checksum fields are present. The decompression path is driven by + * flags: Raw LZO format has flags=0 so all checksum logic is naturally + * skipped. The 9 probe bytes (which belong to the first data block in Raw + * LZO) are cached in peek_buf and consumed first by lzo_read_peek / + * lzo_read_uint32_peek to keep byte alignment. + */ +#include + +/* LZO block buffer size: standard lzop default block size is 256KB */ +#define LZO_BUFFER_SIZE (1<<20) + +/* + * LZO decompression state structure (complete definition; gfile.h only + * declares the pointer). Same heap-allocated double buffer design as + * zlib_stuff / bzlib_stuff. + */ +struct lzo_stuff +{ + int out_size; + int out_pos; + int eof; + unsigned int flags; /* flags from lzop header; 0 for raw LZO */ + bool_t has_lzop_header; /* TRUE=standard lzop, FALSE=raw LZO */ + int peek_size; /* valid bytes in peek_buf */ + int peek_pos; /* current read offset in peek_buf */ + char peek_buf[9]; /* probe buffer (at most 9 bytes) */ + char in[LZO_BUFFER_SIZE]; + char out[LZO_BUFFER_SIZE]; +}; + +/* lzop file magic: \x89 L Z O \x00 \x0d \x0a \x1a \x0a */ +static const unsigned char lzop_magic[9] = { + 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a +}; + +/* lzop header flag bits (from lzop-1.03/src/conf.h) */ +#define LZOP_F_ADLER32_D 0x00000001 /* adler32 checksum of decompressed data */ +#define LZOP_F_ADLER32_C 0x00000002 /* adler32 checksum of compressed data */ +#define LZOP_F_CRC32_D 0x00000100 /* crc32 checksum of decompressed data */ +#define LZOP_F_CRC32_C 0x00000200 /* crc32 checksum of compressed data */ + +/* + * Helper: read exactly n bytes from the underlying file descriptor. + * Only local files are supported here, so this just wraps read_and_retry + * (same as the reads inside gz_file_read / bz_file_read). A short read + * means the file was truncated and is reported to the caller. + */ +static ssize_t +read_block_bytes(gfile_t *fd, void *buf, size_t n) +{ + size_t total = 0; + char *p = (char *) buf; + + while (total < n) + { + ssize_t r = read_and_retry(fd, p + total, n - total); + + if (r == 0) + break; /* EOF, return what we have */ + if (r < 0) + return -1; /* read error */ + total += r; + } + return (ssize_t) total; +} + +/* + * Helper: read one big-endian uint32 from the file. + * All multi-byte integers in the lzop format are big endian. + * Returns 0 on success, -1 on error (read failure or truncation). + */ +static int +read_block_uint32(gfile_t *fd, uint32_t *val) +{ + unsigned char b[4]; + + if (read_block_bytes(fd, b, 4) < 4) + return -1; + *val = ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) | + ((uint32_t) b[2] << 8) | ((uint32_t) b[3]); + return 0; +} + +/* + * peek helper: read from peek_buf first, then fall back to the file. + * + * During format probing, the first bytes of the file may already have been + * consumed. For the standard lzop path peek_buf is always empty, so these + * helpers degenerate to plain file reads. For the Raw LZO path, the probe + * bytes belong to the first data block and are consumed gradually here. + */ +static ssize_t +lzo_read_peek(gfile_t *fd, void *buf, size_t n) +{ + struct lzo_stuff *z = fd->u.lzo; + size_t total = 0; + char *p = (char *) buf; + + /* consume the probe bytes first */ + if (z->peek_pos < z->peek_size) + { + size_t avail = (size_t)(z->peek_size - z->peek_pos); + + if (avail > n) + avail = n; + memcpy(p, z->peek_buf + z->peek_pos, avail); + z->peek_pos += (int) avail; + p += avail; + total += avail; + n -= avail; + } + + /* peek_buf exhausted, read the rest from the file */ + if (n > 0) + { + ssize_t r = read_block_bytes(fd, p, n); + + if (r < 0) + return -1; + total += (size_t) r; + } + + return (ssize_t) total; +} + +/* peek helper: read one big-endian uint32 through the peek mechanism */ +static int +lzo_read_uint32_peek(gfile_t *fd, uint32_t *val) +{ + unsigned char b[4]; + + if (lzo_read_peek(fd, b, 4) < 4) + return -1; + *val = ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) | + ((uint32_t) b[2] << 8) | ((uint32_t) b[3]); + return 0; +} + +/* + * LZO block decompression main loop - lzo_file_read() + * + * Processing flow: + * 1. if there is still decompressed data in out[], return it to the caller + * 2. read the next block header (uncomp_len + comp_len) + * 3. [standard lzop only] read the block checksum fields (driven by flags) + * 4. read comp_len bytes of LZO data into in[] + * 5. decompress with lzo1x_decompress_safe() into out[], verify size + * 6. [standard lzop only] verify checksums of the decompressed data + * 7. return the out[] data to the caller + * + * Protection: + * - block size sanity checks (LZO_BUFFER_SIZE cap, compression bomb guard) + * - comp_len > uncomp_len rejected (compressed data larger than the + * original is impossible; equal means incompressible data stored raw) + * - lzo1x_decompress_safe (bounds-checked) instead of the unsafe variant + * - decompressed size cross-checked against the header's uncomp_len + * - [standard lzop only] adler32/crc32 checksum verification + */ +static ssize_t +lzo_file_read(gfile_t *fd, void *ptr, size_t len) +{ + struct lzo_stuff *z = fd->u.lzo; + + if (!z) + return -1; /* defensive: uninitialized call */ + + for (;;) + { + /* + * Step 1: if there is leftover decompressed data in out[], + * return it. Same pattern as gz_file_read / bz_file_read. + */ + if (z->out_pos < z->out_size || z->eof) + { + size_t avail = z->out_size - z->out_pos; + + if (avail > 0) + { + if (avail > len) + avail = len; + memcpy(ptr, z->out + z->out_pos, avail); + z->out_pos += (int) avail; + return (ssize_t) avail; + } + if (z->eof) + return 0; /* end of file, no more data */ + } + + /* output buffer exhausted, prepare to read the next block */ + z->out_size = 0; + z->out_pos = 0; + + /* + * Step 2: read the block header - uncompressed and compressed + * sizes (4 bytes each, big endian). Use the peek variants so + * that the Raw LZO probe bytes are consumed in order. + */ + uint32_t uncomp_len, comp_len; + + if (lzo_read_uint32_peek(fd, &uncomp_len) < 0) + return -1; /* read error or truncation */ + if (uncomp_len == 0) + { + z->eof = 1; + return 0; /* EOF marker block - normal end of file */ + } + if (lzo_read_uint32_peek(fd, &comp_len) < 0) + return -1; + + /* sanity checks on the block sizes */ + if (comp_len > (uint32_t) LZO_BUFFER_SIZE) + { + gfile_printf_then_putc_newline("lzo block too large: %u", comp_len); + return -1; + } + if (comp_len == 0 && uncomp_len > 0) + { + /* comp_len 0 with non-zero uncomp_len - corrupted header */ + gfile_printf_then_putc_newline("lzo corrupted block: comp_len=0, uncomp_len=%u", + uncomp_len); + return -1; + } + if (comp_len > uncomp_len) + { + /* compressed data larger than the original - not LZO data */ + gfile_printf_then_putc_newline("lzo block comp > uncomp"); + return -1; + } + if (uncomp_len > (uint32_t) LZO_BUFFER_SIZE) + { + gfile_printf_then_putc_newline("lzo uncompressed size too large: %u", + uncomp_len); + return -1; + } + + /* + * Step 3: read and record the block checksums [standard lzop only]. + * Raw LZO: flags=0, all conditions are false, so this is skipped. + */ + uint32_t expected_c_adler32 = 0, expected_c_crc32 = 0; + uint32_t expected_d_adler32 = 0, expected_d_crc32 = 0; + + if (z->flags & LZOP_F_ADLER32_D) + { + if (lzo_read_uint32_peek(fd, &expected_d_adler32) < 0) + return -1; + } + if (z->flags & LZOP_F_CRC32_D) + { + if (lzo_read_uint32_peek(fd, &expected_d_crc32) < 0) + return -1; + } + if (z->flags & LZOP_F_ADLER32_C) + { + if (lzo_read_uint32_peek(fd, &expected_c_adler32) < 0) + return -1; + } + if (z->flags & LZOP_F_CRC32_C) + { + if (lzo_read_uint32_peek(fd, &expected_c_crc32) < 0) + return -1; + } + + /* + * C-checksums (expected_c_adler32 / expected_c_crc32) are consumed + * from the stream to keep the pointer aligned but not verified: + * the D-checksum already catches any error the C-checksum would + * catch (a corrupt compressed block must produce corrupt output), + * and verifying only D-checksums is faster. + */ + (void) expected_c_adler32; + (void) expected_c_crc32; + + /* + * Step 4: read the compressed data block (comp_len bytes). + * The file pointer has skipped all checksum fields by now. + * A short read means the block was truncated. + */ + if (lzo_read_peek(fd, z->in, comp_len) < (ssize_t) comp_len) + return -1; + + /* + * Step 5: decompress. + * comp_len < uncomp_len -> data was LZO-compressed + * comp_len == uncomp_len -> incompressible data stored raw + */ + if (comp_len < uncomp_len) + { + lzo_uint d = uncomp_len; + int r = lzo1x_decompress_safe((const lzo_bytep) z->in, comp_len, + (lzo_bytep) z->out, &d, NULL); + + if (r != LZO_E_OK) + { + /* decompression failure - corrupt data or not LZO */ + gfile_printf_then_putc_newline("lzo safe decompress failed: %d", r); + return -1; + } + /* decompressed size must match the block header */ + if (d != uncomp_len) + { + gfile_printf_then_putc_newline( + "lzo decompression size mismatch: %lu vs %u", + (unsigned long) d, uncomp_len); + return -1; + } + } + else + { + /* incompressible data stored raw - copy as is */ + memcpy(z->out, z->in, comp_len); + uncomp_len = comp_len; + } + + /* + * Step 6: verify the checksums [standard lzop only]. + * Even a "successful" decompression (no error, matching size) + * can silently corrupt data; the checksums catch that. + */ + if (z->flags & LZOP_F_ADLER32_D) + { + uint32_t computed = lzo_adler32(1, (const lzo_bytep) z->out, + uncomp_len); + + if (computed != expected_d_adler32) + { + gfile_printf_then_putc_newline( + "lzo adler32 checksum mismatch " + "(expected 0x%08x, computed 0x%08x)", + expected_d_adler32, computed); + return -1; + } + } + if (z->flags & LZOP_F_CRC32_D) + { + uint32_t computed = lzo_crc32(0, (const lzo_bytep) z->out, + uncomp_len); + + if (computed != expected_d_crc32) + { + gfile_printf_then_putc_newline("lzo crc32 checksum mismatch"); + return -1; + } + } + + /* decompression succeeded, hand out[] to the caller */ + z->out_size = uncomp_len; + } +} + +/* LZO file close: free the lzo_stuff heap buffers */ +static int +lzo_file_close(gfile_t *fd) +{ + if (fd->u.lzo) + { + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + } + return 0; +} + +/* + * lzop header parser - lzo_skip_header() + * + * Parses the standard lzop header, skipping all fields to reach the first + * data block, and saves flags (checksum types etc.) for later use. + * + * magic_already_verified: if the caller has already read and verified the + * magic via probing, pass true to skip the magic; otherwise this function + * reads and verifies it itself. + * + * Protection: + * - every read checks its return value (truncated files) + * - name_len is 1 byte (max 255), path_len capped at 4096 (malicious + * header guard), path is read in 64-byte chunks into the 1024-byte + * stack buffer so it cannot overflow + */ +static int +lzo_skip_header(gfile_t *fd, bool magic_already_verified) +{ + unsigned char hdr_buf[1024]; + unsigned char *buf = hdr_buf; + uint32_t flags; + int name_len; + + if (!magic_already_verified) + { + /* verify the magic: read 9 bytes and compare, reject non-lzop */ + if (read_block_bytes(fd, buf, 9) < 9) + return -1; + if (memcmp(buf, lzop_magic, 9) != 0) + { + gfile_printf_then_putc_newline("not a valid lzop file"); + return -1; + } + } + /* magic_already_verified=true: probe already consumed the magic */ + + /* + * skip version info: version(2) + lib_version(2) + ver_needed(2) + * + method(1) + level(1) = 8 bytes + * These fields do not affect decompression. + */ + if (read_block_bytes(fd, buf, 8) < 8) + return -1; + + /* read flags (4 bytes big endian) and save for block checksum logic */ + if (read_block_uint32(fd, &flags) < 0) + return -1; + fd->u.lzo->flags = flags; + + /* skip mode(4) + mtime_low(4) + mtime_high(4) = 12 bytes */ + if (read_block_bytes(fd, buf, 12) < 12) + return -1; + + /* if F_H_EXTRA_FIELD(0x40) set: skip 1-byte extra version */ + if (flags & 0x00000040) + { + if (read_block_bytes(fd, buf, 1) < 1) + return -1; + } + + /* if F_H_FILTER(0x800) set: skip 4-byte filter ID */ + if (flags & 0x00000800) + { + if (read_block_bytes(fd, buf, 4) < 4) + return -1; + } + + /* read the original file name length (1 byte) and skip the name */ + if (read_block_bytes(fd, buf, 1) < 1) + return -1; + name_len = buf[0]; + if (name_len > 0) + { + if (read_block_bytes(fd, buf, name_len) < name_len) + return -1; + } + + /* if F_H_PATH(0x2000) set: skip the file path (chunked reads) */ + if (flags & 0x00002000) + { + uint32_t path_len; + + if (read_block_uint32(fd, &path_len) < 0) + return -1; + if (path_len > 4096) + { + gfile_printf_then_putc_newline("lzop path too long: %u", path_len); + return -1; + } + while (path_len > 0) + { + uint32_t chunk = (path_len > 64) ? 64 : path_len; + + if (read_block_bytes(fd, buf, chunk) < (ssize_t) chunk) + return -1; + path_len -= chunk; + } + } + + /* skip the header checksum (4 bytes) - skipped, not verified */ + if (read_block_bytes(fd, buf, 4) < 4) + return -1; + + /* header parsed, the file pointer now points at the first data block */ + return 0; +} + +/* + * LZO file open - lzo_file_open() + * + * Initializes the LZO library, allocates lzo_stuff (heap, 2xLZO_BUFFER_SIZE + * plus the probe buffer), auto-detects the file format and wires up + * fd->read / fd->close. + * + * Format auto-detection (probe the first 9 bytes): + * a) magic matches -> standard lzop, lzo_skip_header parses the rest + * b) >= 8 bytes -> Raw LZO, probe bytes cached in peek_buf and + * consumed later (first 8 = uncomp_len + comp_len, + * 9th = first byte of compressed data) + * c) < 8 bytes -> file too short to be valid LZO data + * + * Same calling pattern and error return convention as gz_file_open / + * bz_file_open. + */ +static int +lzo_file_open(gfile_t *fd) +{ + /* initialize the LZO library (version check, idempotent) */ + if (lzo_init() != LZO_E_OK) + { + gfile_printf_then_putc_newline("lzo_init() failed"); + return 1; + } + + /* allocate the decompression state structure (heap) */ + if (!(fd->u.lzo = gfile_malloc(sizeof *fd->u.lzo))) + { + gfile_printf_then_putc_newline("Out of memory"); + return 1; + } + memset(fd->u.lzo, 0, sizeof *fd->u.lzo); + + /* format auto-detection: probe the first 9 bytes, compare magic */ + { + unsigned char probe[9]; + ssize_t n = read_block_bytes(fd, probe, 9); + + if (n >= 9 && memcmp(probe, lzop_magic, 9) == 0) + { + /* path A: standard lzop container format (magic verified) */ + fd->u.lzo->has_lzop_header = TRUE; + + /* parse the remaining header (skip the magic) */ + if (lzo_skip_header(fd, TRUE) != 0) + { + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + return 1; + } + } + else if (n >= 8) + { + /* + * path B: Raw LZO format - no magic, header or checksums. + * The n probe bytes belong to the first data block; cache + * them in peek_buf for lzo_read_*_peek to consume later. + * flags stays 0 (memset), so all checksum logic is skipped. + */ + fd->u.lzo->has_lzop_header = FALSE; + fd->u.lzo->flags = 0; + fd->u.lzo->peek_size = (int) n; + fd->u.lzo->peek_pos = 0; + memcpy(fd->u.lzo->peek_buf, probe, (size_t) n); + } + else + { + /* file too short (less than 8 bytes) to be valid LZO data */ + gfile_printf_then_putc_newline("lzo file too short (%ld bytes)", (long) n); + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + return 1; + } + } + + /* register callbacks: gfile_read/gfile_close will use our functions */ + fd->read = lzo_file_read; + fd->close = lzo_file_close; return 0; } #endif @@ -1296,6 +1894,22 @@ int gfile_open(gfile_t* fd, const char* fpath, int flags, int* response_code, co } return zstd_file_open(fd); +#endif + } + else if (s && strcasecmp(s, ".lzo") == 0) + { +#ifndef USE_LZO + gfile_printf_then_putc_newline(".lzo not supported"); +#else + if (flags != GFILE_OPEN_FOR_READ) + { + gfile_printf_then_putc_newline(".lzo not yet supported for writable tables"); + *response_code = 415; + *response_string = "Unsupported File Type"; + return 1; + } + fd->compression = LZO_COMPRESSION; + return lzo_file_open(fd); #endif } else if (s && strcasecmp(s,".z") == 0) @@ -1330,7 +1944,8 @@ gfile_close(gfile_t*fd) * for the compressed data implementation we need to call the "close" callback. Other implementations * didn't use to call this callback here and it will remain so. */ - if (fd->compression == GZ_COMPRESSION || fd->compression == ZSTD_COMPRESSION) + if (fd->compression == GZ_COMPRESSION || fd->compression == ZSTD_COMPRESSION || + fd->compression == LZO_COMPRESSION) { fd->close(fd); } diff --git a/src/include/fstream/gfile.h b/src/include/fstream/gfile.h index 565a7d910c9..07ddfd9da0c 100644 --- a/src/include/fstream/gfile.h +++ b/src/include/fstream/gfile.h @@ -11,6 +11,9 @@ #ifdef USE_ZSTD #include #endif +#ifdef USE_LZO +#include +#endif #ifdef WIN32 #include @@ -45,9 +48,17 @@ typedef enum Compression_type NO_COMPRESSION = 0, GZ_COMPRESSION, BZ_COMPRESSION, - ZSTD_COMPRESSION + ZSTD_COMPRESSION, + LZO_COMPRESSION } compression_type; +/* + * LZO decompression state (incomplete declaration, complete definition + * in gfile.c), following the same pattern as zlib_stuff/bzlib_stuff: + * heap-allocated double buffer + EOF flag. + */ +struct lzo_stuff; + /* The struct gfile_t is private. Please do not use any of its fields. */ typedef struct gfile_t { @@ -77,6 +88,9 @@ typedef struct gfile_t #endif #ifdef USE_ZSTD struct zstdlib_stuff*zstd; +#endif +#ifdef USE_LZO + struct lzo_stuff*lzo; #endif }u; bool_t is_write; diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 72a27f363b4..a105f484373 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -72,7 +72,6 @@ reference if 'false' (always true in GPDB) */ #undef FLOAT8PASSBYVAL - /* Define to 1 to build ORCA in debug (GPOS_DEBUG=1) */ #undef GPOS_DEBUG @@ -211,10 +210,6 @@ /* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ #undef HAVE_FSEEKO -/* Define to 1 if you have __sync_lock_test_and_set(int *) and friends. */ -#undef HAVE_GCC_INT_ATOMICS - - /* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */ #undef HAVE_GCC__ATOMIC_INT32_CAS @@ -364,6 +359,9 @@ /* Define to 1 if you have the `ssl' library (-lssl). */ #undef HAVE_LIBSSL +/* Define to 1 if you have the `uring' library (-luring). */ +#undef HAVE_LIBURING + /* Define to 1 if you have the `uv' library (-luv). */ #undef HAVE_LIBUV @@ -430,18 +428,9 @@ /* Define to 1 if you have the `ppoll' function. */ #undef HAVE_PPOLL -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - /* Define if protobuf is available */ #undef HAVE_PROTOBUF -/* Define to 1 if you have the `pstat' function. */ -#undef HAVE_PSTAT - -/* Define to 1 if the PS_STRINGS thing exists. */ -#undef HAVE_PS_STRINGS - /* Define if you have POSIX threads libraries and header files. */ #undef HAVE_PTHREAD @@ -631,17 +620,11 @@ /* Define to 1 if you have the `wcstombs_l' function. */ #undef HAVE_WCSTOMBS_L -/* Define to 1 if you have the header file. */ -#undef HAVE_WCTYPE_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_WINLDAP_H - /* Define to 1 if you have the header file. */ #undef HAVE_WINSOCK2_H -/* Define to 1 if you have the `writev' function. */ -#undef HAVE_WRITEV +/* Define to 1 if you have the `X509_get_signature_info' function. */ +#undef HAVE_X509_GET_SIGNATURE_INFO /* Define to 1 if you have the `X509_get_signature_nid' function. */ #undef HAVE_X509_GET_SIGNATURE_NID @@ -655,9 +638,6 @@ /* Define if zstd is available */ #undef HAVE_ZSTD -/* Define to 1 if the system has the type `_Bool'. */ -#undef HAVE__BOOL - /* Define to 1 if your compiler understands __builtin_bswap16. */ #undef HAVE__BUILTIN_BSWAP16 @@ -703,15 +683,6 @@ /* Define to 1 if your compiler understands _Static_assert. */ #undef HAVE__STATIC_ASSERT -/* Define to 1 if you have the `etcdlib' library (-letcdlib). */ -#undef HAVE_LIBETCDLIB - -/* Define to 1 if you have the `__strtoll' function. */ -#undef HAVE___STRTOLL - -/* Define to 1 if you have the `__strtoull' function. */ -#undef HAVE___STRTOULL - /* Define to the appropriate printf length modifier for 64-bit ints. */ #undef INT64_MODIFIER @@ -886,12 +857,15 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 -/* Define to 1 to build with MDB locales. (--with-mdblocales) */ -#undef USE_MDBLOCALES +/* Define to 1 to build with LZO support. (--with-lzo) */ +#undef USE_LZO /* Define to 1 to build with Mapreduce capabilities (--enable-mapreduce) */ #undef USE_MAPREDUCE +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES @@ -938,7 +912,7 @@ /* Define to select Win32-style shared memory. */ #undef USE_WIN32_SHARED_MEMORY -/* Define to 1 to build with ZSTD support. (--with-zstd) */ +/* Define to build with zstd support. (--with-zstd) */ #undef USE_ZSTD /* Define to 1 if `wcstombs_l' requires . */