Add more tests (#94)

* test Z_NEED_DICT error

* test large input sizes

* test reading one byte at a time

* fix test for 1.6

Author: nhz2

test/Project.toml

@@ -4,3 +4,6 @@ CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestsForCodecPackages = "c2e61002-3542-480d-8b3c-5f05cc4f8554"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+
+[sources]
+CodecZlib = {path = ".."}

test/big-mem-tests.jl

@@ -0,0 +1,61 @@
+# This file contains tests that require a large amount of memory (at least 25 GB)
+# and take a long time to run. The tests are designed to check the 
+# compression and decompression functionality of the package 
+# with very large inputs. These tests are not run with CI
+
+using Test
+using CodecZlib
+
+# Enable this when https://github.com/JuliaIO/CodecZlib.jl/issues/88 is fixed.
+# @testset "memory leak" begin
+#     function foo()
+#         for i in 1:1000000
+#             c = transcode(GzipCompressor(), zeros(UInt8,16))
+#             u = transcode(GzipDecompressor(), c)
+#         end
+#     end
+#     foo()
+# end
+
+@testset "Big Memory Tests" begin
+    Sys.WORD_SIZE == 64 || error("tests require 64 bit word size")
+    @info "compressing zeros"
+    for n in (2^32 - 1, 2^32, 2^32 +1)
+        @info "compressing"
+        local c = transcode(GzipCompressor, zeros(UInt8, n))
+        @info "decompressing"
+        local u = transcode(GzipDecompressor, c)
+        c = nothing
+        all_zero = all(iszero, u)
+        len_n = length(u) == n
+        @test all_zero && len_n
+    end
+
+    @info "compressing random"
+    for n in (2^32 - 1, 2^32, 2^32 +1)
+        local u = rand(UInt8, n)
+        @info "compressing"
+        local c = transcode(GzipCompressor, u)
+        @info "decompressing"
+        local u2 = transcode(GzipDecompressor, c)
+        c = nothing
+        are_equal = u == u2
+        @test are_equal
+    end
+
+    @info "decompressing huge concatenation"
+    uncompressed = rand(UInt8, 2^20)
+    @info "compressing"
+    compressed = transcode(GzipCompressor, uncompressed)
+    total_compressed = UInt8[]
+    sizehint!(total_compressed, length(compressed)*2^12)
+    total_uncompressed = UInt8[]
+    sizehint!(total_uncompressed, length(uncompressed)*2^12)
+    for i in 1:2^12
+        append!(total_uncompressed, uncompressed)
+        append!(total_compressed, compressed)
+    end
+    @test length(total_compressed) > 2^32
+    @info "decompressing"
+    @test total_uncompressed == transcode(GzipDecompressor, total_compressed)
+end

test/runtests.jl

@@ -20,6 +20,19 @@ Aqua.test_all(CodecZlib)
 
 const testdir = @__DIR__
 
+# decompress one byte at a time
+function decompress_bytes(decoder, data::Vector{UInt8})::Vector{UInt8}
+    io = IOBuffer()
+    s = decoder(io; bufsize=1)
+    for i in eachindex(data)
+        write(s, data[i])
+        flush(s)
+    end
+    write(s, TranscodingStreams.TOKEN_END)
+    flush(s)
+    take!(io)
+end
+
 @testset "Gzip Codec" begin
     codec = GzipCompressor()
     @test codec isa GzipCompressor
@@ -211,8 +224,7 @@ end
     @test codec isa DeflateCompressor
     @test occursin(r"^(CodecZlib\.)?DeflateCompressor\(level=-1, windowbits=-\d+\)$", sprint(show, codec))
     @test CodecZlib.initialize(codec) === nothing
-    # FIXME: This test fails.
-    #@test CodecZlib.finalize(codec) === nothing
+    @test CodecZlib.finalize(codec) === nothing
 
     codec = DeflateDecompressor()
     @test codec isa DeflateDecompressor
@@ -233,6 +245,20 @@ end
     @test_throws ArgumentError DeflateCompressor(level=10)
     @test_throws ArgumentError DeflateCompressor(windowbits=16)
     @test_throws ArgumentError DeflateDecompressor(windowbits=16)
+
+    # Test decoding byte by byte
+    # Exercise Deflate distances and lengths
+    for len in [10, 100, 200, 257, 258, 259]
+        thing = rand(UInt8, len)
+        d = UInt8[]
+        for dist in [0:258; 1000:1030; 2000:1000:33000;]
+            append!(d, thing)
+            append!(d, rand(0x00:0x0f, dist))
+        end
+        c = transcode(DeflateCompressor, d)
+        @test transcode(DeflateDecompressor, c) == d
+        @test decompress_bytes(DeflateDecompressorStream, c) == d
+    end
 end
 
 # Test APIs of TranscodingStreams.jl using the gzip compressor/decompressor.
@@ -329,6 +355,17 @@ end
     local compressed = transcode(ZlibCompressor, uncompressed)
     compressed[70] ⊻= 0x01
     @test_throws ZlibError transcode(ZlibDecompressor, compressed)
+    # Z_NEED_DICT error
+    try
+        transcode(
+            ZlibDecompressor,
+            UInt8[0x78, 0xbb, 0x00, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01],
+        )
+        @test false
+    catch e
+        @test e isa ZlibError
+        @test endswith(e.msg, "(code: $(CodecZlib.Z_NEED_DICT))")
+    end
 end
 @testset "error printing" begin
     @test sprint(Base.showerror, ZlibError("test error message")) ==