Monomorphic / differential clj-http tests for comparison

project.clj

@@ -2,7 +2,7 @@
 
 (defproject aleph (or (System/getenv "PROJECT_VERSION") "0.5.0")
   :description "A framework for asynchronous communication"
-  :repositories {"jboss" "https://repository.jboss.org/nexus/content/groups/public/"
+  :repositories {"jboss"               "https://repository.jboss.org/nexus/content/groups/public/"
                  "sonatype-oss-public" "https://oss.sonatype.org/content/groups/public/"}
   :url "https://github.com/clj-commons/aleph"
   :license {:name "MIT License"}
@@ -21,39 +21,51 @@
                  [io.netty/netty-handler-proxy ~netty-version]
                  [io.netty/netty-resolver ~netty-version]
                  [io.netty/netty-resolver-dns ~netty-version]]
-  :profiles {:dev  {:dependencies [[org.clojure/clojure "1.10.3"]
-                                   [criterium "0.4.6"]
-                                   [cheshire "5.10.0"]
-                                   [org.slf4j/slf4j-simple "1.7.30"]
-                                   [com.cognitect/transit-clj "1.0.324"]
-                                   [spootnik/signal "0.2.4"]
-                                   [me.mourjo/dynamic-redef "0.1.0"]]}
+  :profiles {:dev          {:dependencies [[org.clojure/clojure "1.11.1"]
+                                           [criterium "0.4.6"]
+                                           [cheshire "5.10.0"]
+                                           [org.slf4j/slf4j-simple "1.7.30"]
+                                           [com.cognitect/transit-clj "1.0.324"]
+                                           [spootnik/signal "0.2.4"]
+                                           [me.mourjo/dynamic-redef "0.1.0"]
+
+                                           ;; for testing clj-http parity
+                                           [clj-http "3.12.3"]
+                                           [ring/ring-jetty-adapter "1.9.3"]
+                                           [org.apache.logging.log4j/log4j-api "2.17.1"]
+                                           [org.apache.logging.log4j/log4j-core "2.17.1"]
+                                           [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]]}
              :lein-to-deps {:source-paths ["deps"]}
              ;; This is for self-generating certs for testing ONLY:
-             :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"]
-                                   [org.bouncycastle/bcpkix-jdk15on "1.69"]]
-                    :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}}
-  :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/"
+             :test         {:dependencies  [[org.bouncycastle/bcprov-jdk15on "1.69"]
+                                            [org.bouncycastle/bcpkix-jdk15on "1.69"]]
+                            :javac-options ^:replace ["--release" "12"] ; necessary for some tests
+                            :jvm-opts      ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}}
+  :codox {:src-dir-uri               "https://github.com/ztellman/aleph/tree/master/"
           :src-linenum-anchor-prefix "L"
-          :defaults {:doc/format :markdown}
-          :include [aleph.tcp
-                    aleph.udp
-                    aleph.http
-                    aleph.flow]
-          :output-dir "doc"}
+          :defaults                  {:doc/format :markdown}
+          :include                   [aleph.tcp
+                                      aleph.udp
+                                      aleph.http
+                                      aleph.flow]
+          :output-dir                "doc"}
   :plugins [[lein-codox "0.10.7"]
-            [lein-jammin "0.1.1"]
             [lein-marginalia "0.9.1"]
             [lein-pprint "1.3.2"]
             [ztellman/lein-cljfmt "0.1.10"]]
   :java-source-paths ["src/aleph/utils"]
   :cljfmt {:indents {#".*" [[:inner 0]]}}
-  :test-selectors {:default #(not
-                               (some #{:benchmark :stress}
-                                 (cons (:tag %) (keys %))))
-                   :benchmark :benchmark
-                   :stress :stress
-                   :all (constantly true)}
+  :test-selectors {:default     #(not
+                                   (some #{:benchmark :stress :integration :ignore}
+                                     (cons (:tag %) (keys %))))
+                   :benchmark   :benchmark
+                   :integration :integration
+                   :stress      :stress
+                   :clj-http    [(fn clj-http-ns-pred [namespc & _]
+                                   (.contains (str namespc) "clj-http"))
+                                 (fn clj-http-test-pred [m & _]
+                                   (not (:ignore m)))]
+                   :all         (constantly true)}
   :jvm-opts ^:replace ["-server"
                        "-Xmx2g"
                        "-XX:+HeapDumpOnOutOfMemoryError"

src/aleph/http.clj

@@ -102,7 +102,7 @@
    | `max-queue-size` | the maximum number of pending acquires from the pool that are allowed before `acquire` will start to throw a `java.util.concurrent.RejectedExecutionException`, defaults to `65536`
    | `control-period` | the interval, in milliseconds, between use of the controller to adjust the size of the pool, defaults to `60000`
    | `dns-options` | an optional map with async DNS resolver settings, for more information check `aleph.netty/dns-resolver-group`. When set, ignores `name-resolver` setting from `connection-options` in favor of shared DNS resolver instance
-   | `middleware` | a function to modify request before sending, defaults to `aleph.http.client-middleware/wrap-request`
+   | `middleware` | a function to modify clients/requests before sending, defaults to `aleph.http.client-middleware/wrap-request`
    | `pool-builder-fn` | an optional one arity function which returns a `io.aleph.dirigiste.IPool` from a map containing the following keys: `generate`, `destroy`, `control-period`, `max-queue-length` and `stats-callback`.
    | `pool-controller-builder-fn` | an optional zero arity function which returns a `io.aleph.dirigiste.IPool$Controller`.
 

src/aleph/http/client_middleware.clj

@@ -155,6 +155,7 @@
     {:scheme (keyword (.getProtocol url-parsed))
      :server-name (.getHost url-parsed)
      :server-port (when-pos (.getPort url-parsed))
+     :url url
      :uri (url-encode-illegal-characters (.getPath url-parsed))
      :user-info (when-let [user-info (.getUserInfo url-parsed)]
                   (URLDecoder/decode user-info))
@@ -243,12 +244,13 @@
       (if (unexceptional-status? status)
         rsp
         (cond
-
           (false? (opt req :throw-exceptions))
           rsp
 
           (instance? InputStream body)
-          (d/chain' (d/future (bs/to-byte-array body))
+          (d/chain'
+            (d/future
+              (bs/to-byte-array body))
             (fn [body]
               (d/error-deferred
                 (ex-info
@@ -563,9 +565,8 @@
   [req]
   (if-let [url (:url req)]
     (-> req
-      (dissoc :url)
-      (assoc :request-url url)
-      (merge (parse-url url)))
+        (assoc :request-url url)
+        (merge (parse-url url)))
     req))
 
 (defn wrap-request-timing
@@ -918,7 +919,13 @@
       (opt req :save-request)
       (assoc :aleph/request req'))))
 
+(def default-client-middleware
+  "Default middleware that takes a client fn"
+  [wrap-exceptions
+   wrap-request-timing])
+
 (def default-middleware
+  "Default middleware that takes a request map"
   [wrap-method
    wrap-url
    wrap-nested-params
@@ -937,22 +944,24 @@
   "Returns a batteries-included HTTP request function corresponding to the given
   core client. See default-middleware for the middleware wrappers that are used
   by default"
-  [client]
-  (let [client' (-> client
-                  wrap-exceptions
-                  wrap-request-timing)]
-    (fn [req]
-      (let [executor (ex/executor)]
-        (if (:aleph.http.client/close req)
-          (client req)
-
-          (let [req' (reduce #(%2 %1) req default-middleware)]
-            (d/chain' (client' req')
-
-              ;; coerce the response body
-              (fn [{:keys [body] :as rsp}]
-                (let [rsp' (handle-response-debug req' rsp)]
-                  (if (and (some? body) (some? (:as req')))
-                    (d/future-with (or executor (ex/wait-pool))
-                      (coerce-response-body req' rsp'))
-                    rsp'))))))))))
+  ([client]
+   (wrap-request client default-client-middleware default-middleware))
+  ([client client-middleware middleware]
+   (let [client' (reduce #(%2 %1)
+                         client
+                         client-middleware)]
+     (fn [req]
+       (let [executor (ex/executor)]
+         (if (:aleph.http.client/close req)
+           (client req)
+
+           (let [req' (reduce #(%2 %1) req middleware)]
+             (d/chain' (client' req')
+
+                       ;; coerce the response body
+                       (fn [{:keys [body] :as rsp}]
+                         (let [rsp' (handle-response-debug req' rsp)]
+                           (if (and (some? body) (some? (:as req')))
+                             (d/future-with (or executor (ex/wait-pool))
+                               (coerce-response-body req' rsp'))
+                             rsp')))))))))))

src/aleph/http/core.clj

@@ -224,9 +224,10 @@
   :remote-addr (netty/channel-remote-address ch))
 
 (p/def-derived-map NettyResponse [^HttpResponse rsp complete body]
-  :status (-> rsp .status .code)
+  :status (-> rsp (.status) (.code))
+  :reason-phrase (-> rsp (.status) (.reasonPhrase))
   :aleph/keep-alive? (HttpUtil/isKeepAlive rsp)
-  :headers (-> rsp .headers headers->map)
+  :headers (-> rsp (.headers) headers->map)
   :aleph/complete complete
   :body body)
 

src/aleph/http/multipart.clj

@@ -201,7 +201,7 @@
 
 (defn decode-request
   "Takes a ring request and returns a manifold stream which yields
-   parts of the mutlipart/form-data encoded body. In case the size of
+   parts of the multipart/form-data encoded body. In case the size of
    a part content exceeds `:memory-limit` limit (16KB by default),
    corresponding payload would be written to a temp file. Check `:memory?`
    flag to know whether content might be read directly from `:content` or

test-resources/big_array_json.json

@@ -0,0 +1,102 @@
+[
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+  {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}
+]

test-resources/m.txt

@@ -0,0 +1,4 @@
+this
+is
+some
+file.