Add gzip for assets as well as server responses (#983)

* added static_compression_handler

* fixed a couple typos in static_compression_handler_spec

* actually test value of etag in static_compression_handler

* add gzipping to Lucky::TextResponse

* add gzip setting to Lucky::Server

* integrate static compression handler with dynamic compression handler

* dry up static compression handler a little

* Handle content types with additional information

* add application/javascript to gzip_content_types default setting

* added some docs for compression

* Update src/lucky/static_compression_handler.cr

Add explicit return type for should_compress?

Co-Authored-By: Paul Smith <[email protected]>

* static_compression_handler doc clarification

* fix order of spec

* clarified StaticCompressionHandler specs

* add spec for not gzipping based on content type

* tweak static compression handler spec

spec/lucky/static_compression_handler_spec.cr

@@ -0,0 +1,110 @@
+require "../spec_helper"
+require "http"
+
+include ContextHelper
+
+private PATH = "example.css"
+
+describe Lucky::StaticCompressionHandler do
+  it "calls next when not enabled" do
+    context = build_context(path: PATH)
+    context.request.headers["Accept-Encoding"] = "gzip"
+    next_called = false
+
+    call_handler_with(context) { next_called = true }
+
+    next_called.should be_true
+    context.response.headers["Content-Encoding"]?.should_not eq "gzip"
+  end
+
+  it "calls next when content type isn't in Lucky::Server.gzip_content_types" do
+    Lucky::Server.temp_config(gzip_enabled: true, gzip_content_types: %w(text/html)) do
+      context = build_context(path: PATH)
+      context.request.headers["Accept-Encoding"] = "gzip"
+      next_called = false
+
+      call_handler_with(context) { next_called = true }
+
+      next_called.should be_true
+      context.response.headers["Content-Encoding"]?.should_not eq "gzip"
+    end
+  end
+
+  it "delivers the precompressed file when enabled" do
+    Lucky::Server.temp_config(gzip_enabled: true) do
+      output = IO::Memory.new
+      context = build_context_with_io(output, path: PATH)
+
+      context.request.method = "GET"
+      context.request.headers["Accept-Encoding"] = "gzip"
+
+      next_called = false
+      call_handler_with(context) { next_called = true }
+
+      next_called.should be_false
+
+      context.response.headers["Content-Encoding"].should eq "gzip"
+      context.response.headers["Etag"].should eq etag
+      output.close
+      output.to_s.ends_with?(File.read(gzip_path)).should be_true
+    end
+  end
+
+  it "calls next when Accept-Encoding doesn't include gzip" do
+    Lucky::Server.temp_config(gzip_enabled: true) do
+      context = build_context(path: PATH)
+      context.request.headers["Accept-Encoding"] = "whatever"
+      next_called = false
+
+      call_handler_with(context) { next_called = true }
+
+      next_called.should be_true
+      context.response.headers["Content-Encoding"]?.should_not eq "gzip"
+    end
+  end
+
+  it "sends NOT_MODIFIED when file hasn't been modified" do
+    Lucky::Server.temp_config(gzip_enabled: true) do
+      first_context = build_context(path: PATH)
+      first_context.request.method = "GET"
+      first_context.request.headers["Accept-Encoding"] = "gzip"
+
+      call_handler_with(first_context) { }
+
+      last_modified = HTTP.parse_time(first_context.response.headers["Last-Modified"]).not_nil!
+
+      context = build_context(path: PATH)
+      context.request.headers["Accept-Encoding"] = "gzip"
+      context.request.headers["If-Modified-Since"] = HTTP.format_time(last_modified + 1.hour)
+      next_called = false
+
+      call_handler_with(context) { next_called = true }
+
+      next_called.should be_false
+      context.response.status.should eq HTTP::Status::NOT_MODIFIED
+    end
+  end
+end
+
+private def public_dir
+  File.expand_path("spec/fixtures")
+end
+
+private def gzip_path
+  File.join(public_dir, "#{PATH}.gz")
+end
+
+private def etag
+  %{W/"#{last_modified.to_unix}"}
+end
+
+private def last_modified
+  File.info(gzip_path).modification_time
+end
+
+private def call_handler_with(context : HTTP::Server::Context, &block)
+  handler = Lucky::StaticCompressionHandler.new(public_dir: public_dir)
+  handler.next = ->(_ctx : HTTP::Server::Context) { block.call }
+  handler.call(context)
+  context.response.close
+end

spec/lucky/text_response_spec.cr

@@ -30,13 +30,43 @@ describe Lucky::TextResponse do
       context.request.body.to_s.should eq ""
       context.response.status_code.should eq 200
     end
+
+    it "gzips if enabled" do
+      Lucky::Server.temp_config(gzip_enabled: true) do
+        output = IO::Memory.new
+        context = build_context_with_io(output)
+        context.request.headers["Accept-Encoding"] = "gzip"
+
+        print_response_with_body(context, status: 200, body: "some body")
+        context.response.close
+
+        context.response.headers["Content-Encoding"].should eq "gzip"
+        expected_io = IO::Memory.new
+        Gzip::Writer.open(expected_io) { |gzw| gzw.print "some body" }
+        output.to_s.ends_with?(expected_io.to_s).should be_true
+      end
+    end
+
+    it "doesn't gzip when content type isn't in Lucky::Server.gzip_content_types" do
+      Lucky::Server.temp_config(gzip_enabled: true) do
+        output = IO::Memory.new
+        context = build_context_with_io(output)
+        context.request.headers["Accept-Encoding"] = "gzip"
+
+        print_response_with_body(context, status: 200, body: "some body", content_type: "foo/bar")
+        context.response.close
+
+        context.response.headers["Content-Encoding"]?.should_not eq "gzip"
+        output.to_s.ends_with?("some body").should be_true
+      end
+    end
   end
 end
 
 private def print_response(context : HTTP::Server::Context, status : Int32?)
   print_response_with_body(context, "", status)
 end
 
-private def print_response_with_body(context : HTTP::Server::Context, body : String, status : Int32?)
-  Lucky::TextResponse.new(context, "", body, status: status).print
+private def print_response_with_body(context : HTTP::Server::Context, body = "", status = 200, content_type = "text/html")
+  Lucky::TextResponse.new(context, content_type, body, status: status).print
 end

src/lucky/server.cr

@@ -1,8 +1,27 @@
+# Class for configuring server settings
+#
+# The settings created here can be customized in each Lucky app by modifying them in your config/server.cr
 class Lucky::Server
   Habitat.create do
     setting secret_key_base : String
     setting host : String
     setting port : Int32
     setting asset_host : String = ""
+    setting gzip_enabled : Bool = false
+    setting gzip_content_types : Array(String) = %w(
+      application/json
+      application/javascript
+      application/xml
+      font/otf
+      font/ttf
+      font/woff
+      font/woff2
+      image/svg+xml
+      text/css
+      text/csv
+      text/html
+      text/javascript
+      text/plain
+    )
   end
 end

src/lucky/static_compression_handler.cr

@@ -0,0 +1,88 @@
+# Middleware that serves static files that have been pre-compressed.
+# There can be multiple instances and the first in the middleware stack will take precedence.
+# For example, if you want to serve brotli compressed assets for browsers that support it and
+# serve gzip assets for those that don't you would do something like this in your middleware
+# in `src/app_server.cr`:
+#
+# ```
+# [
+#   # ...
+#   Lucky::StaticCompressionHandler.new("./public", file_ext: "br", content_encoding: "br"),
+#   Lucky::StaticCompressionHandler.new("./public", file_ext: "gz", content_encoding: "gzip"),
+#   # ...
+# ]
+# ```
+class Lucky::StaticCompressionHandler
+  include HTTP::Handler
+
+  def initialize(@public_dir : String, @file_ext = "gz", @content_encoding = "gzip")
+  end
+
+  def call(context)
+    original_path = context.request.path.not_nil!
+    request_path = URI.decode(original_path)
+    expanded_path = File.expand_path(request_path, "/")
+    file_path = File.join(@public_dir, expanded_path)
+    compressed_path = "#{file_path}.#{@file_ext}"
+    content_type = MIME.from_filename(file_path, "application/octet-stream")
+
+    if !should_compress?(file_path, content_type, compressed_path, context.request.headers)
+      call_next(context)
+      return
+    end
+
+    context.response.headers["Content-Encoding"] = @content_encoding
+
+    last_modified = modification_time(compressed_path)
+    add_cache_headers(context.response.headers, last_modified)
+
+    if cache_request?(context, last_modified)
+      context.response.status = :not_modified
+      return
+    end
+
+    context.response.content_type = content_type
+    context.response.content_length = File.size(compressed_path)
+    File.open(compressed_path) do |file|
+      IO.copy(file, context.response)
+    end
+  end
+
+  private def should_compress?(file_path, content_type, compressed_path, request_headers) : Bool
+    Lucky::Server.settings.gzip_enabled &&
+      request_headers.includes_word?("Accept-Encoding", @content_encoding) &&
+      Lucky::Server.settings.gzip_content_types.any? { |ct| content_type.starts_with?(ct) } &&
+      File.exists?(compressed_path)
+  end
+
+  private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil
+    response_headers["Etag"] = etag(last_modified)
+    response_headers["Last-Modified"] = HTTP.format_time(last_modified)
+  end
+
+  private def cache_request?(context : HTTP::Server::Context, last_modified : Time) : Bool
+    # According to RFC 7232:
+    # A recipient must ignore If-Modified-Since if the request contains an If-None-Match header field
+    if if_none_match = context.request.if_none_match
+      match = {"*", context.response.headers["Etag"]}
+      if_none_match.any? { |etag| match.includes?(etag) }
+    elsif if_modified_since = context.request.headers["If-Modified-Since"]?
+      header_time = HTTP.parse_time(if_modified_since)
+      # File mtime probably has a higher resolution than the header value.
+      # An exact comparison might be slightly off, so we add 1s padding.
+      # Static files should generally not be modified in subsecond intervals, so this is perfectly safe.
+      # This might be replaced by a more sophisticated time comparison when it becomes available.
+      !!(header_time && last_modified <= header_time + 1.second)
+    else
+      false
+    end
+  end
+
+  private def etag(modification_time)
+    %{W/"#{modification_time.to_unix}"}
+  end
+
+  private def modification_time(file_path)
+    File.info(file_path).modification_time
+  end
+end

src/lucky/text_response.cr

@@ -1,3 +1,12 @@
+{% if !flag?(:without_zlib) %}
+  require "gzip"
+{% end %}
+
+# Writes the *content_type*, *status*, and *body* to the *context* for text responses.
+#
+# There are two settings in `Lucky::Server.settings` that determine if
+# the text response is gzip encoded; `Lucky::Server.settings.gzip_enabled` and `Lucky::Server.settings.gzip_content_types`.
+# These settings can be adjusted in your Lucky app under config/server.cr
 class Lucky::TextResponse < Lucky::Response
   DEFAULT_STATUS = 200
 
@@ -13,10 +22,24 @@ class Lucky::TextResponse < Lucky::Response
   def print : Nil
     context.response.content_type = content_type
     context.response.status_code = status
+    gzip if should_gzip?
     context.response.print body
   end
 
   def status : Int
     @status || context.response.status_code || DEFAULT_STATUS
   end
+
+  private def gzip
+    context.response.headers["Content-Encoding"] = "gzip"
+    context.response.output = Gzip::Writer.new(context.response.output, sync_close: true)
+  end
+
+  private def should_gzip?
+    {% if !flag?(:without_zlib) %}
+      Lucky::Server.settings.gzip_enabled &&
+        context.request.headers.includes_word?("Accept-Encoding", "gzip") &&
+        Lucky::Server.settings.gzip_content_types.includes?(content_type)
+    {% end %}
+  end
 end