cybercyst/xh
1
0
Fork 0
mirror of https://github.com/ducaale/xh.git synced 2025-05-05 15:32:50 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #403 from zuisong/compress-request-body

Browse Source 
support compress request body
This commit is contained in:
Mohamed Daahir 2025-02-12 08:26:06 +00:00 committed by GitHub
commit 4873dc9b7f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 343 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
completions/_xh
View File

@ -49,12 +49,12 @@ none\:"Disable both coloring and formatting"))' \
'--http-version=[HTTP version to use]:VERSION:(1.0 1.1 2 2-prior-knowledge)' \
'*--resolve=[Override DNS resolution for specific domain to a custom IP]:HOST:ADDRESS:_default' \
'--interface=[Bind to a network interface or local IP address]:NAME:_default' \
'--generate=[Generate shell completions or man pages]:KIND:(complete-bash complete-elvish complete-fish complete-nushell complete-powershell complete-zsh man)' \
'()--generate=[Generate shell completions or man pages]:KIND:(complete-bash complete-elvish complete-fish complete-nushell complete-powershell complete-zsh man)' \
'-j[(default) Serialize data items from the command line as a JSON object]' \
'--json[(default) Serialize data items from the command line as a JSON object]' \
'-f[Serialize data items from the command line as form fields]' \
'--form[Serialize data items from the command line as form fields]' \
'(--raw)--multipart[Like --form, but force a multipart/form-data request even without files]' \
'(--raw -x --compress)--multipart[Like --form, but force a multipart/form-data request even without files]' \
'-h[Print only the response headers. Shortcut for --print=h]' \
'--headers[Print only the response headers. Shortcut for --print=h]' \
'-b[Print only the response body. Shortcut for --print=b]' \
@ -69,6 +69,8 @@ none\:"Disable both coloring and formatting"))' \
'*--quiet[Do not print to stdout or stderr]' \
'-S[Always stream the response body]' \
'--stream[Always stream the response body]' \
'*-x[Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate]' \
'*--compress[Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate]' \
'-d[Download the body to a file instead of printing it]' \
'--download[Download the body to a file instead of printing it]' \
'-c[Resume an interrupted download. Requires --download and --output]' \
@ -108,6 +110,7 @@ none\:"Disable both coloring and formatting"))' \
'--no-history-print[]' \
'--no-quiet[]' \
'--no-stream[]' \
'--no-compress[]' \
'--no-output[]' \
'--no-download[]' \
'--no-continue[]' \
@ -142,7 +145,7 @@ none\:"Disable both coloring and formatting"))' \
'--no-help[]' \
'-V[Print version]' \
'--version[Print version]' \
'::raw_method_or_url -- The request URL, preceded by an optional HTTP method:_default' \
':raw_method_or_url -- The request URL, preceded by an optional HTTP method:_default' \
'*::raw_rest_args -- Optional key-value pairs to be included in the request.:_default' \
&& ret=0
}

3
completions/_xh.ps1
View File

@ -72,6 +72,8 @@ Register-ArgumentCompleter -Native -CommandName 'xh' -ScriptBlock {
[CompletionResult]::new('--quiet', '--quiet', [CompletionResultType]::ParameterName, 'Do not print to stdout or stderr')
[CompletionResult]::new('-S', '-S ', [CompletionResultType]::ParameterName, 'Always stream the response body')
[CompletionResult]::new('--stream', '--stream', [CompletionResultType]::ParameterName, 'Always stream the response body')
[CompletionResult]::new('-x', '-x', [CompletionResultType]::ParameterName, 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate')
[CompletionResult]::new('--compress', '--compress', [CompletionResultType]::ParameterName, 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate')
[CompletionResult]::new('-d', '-d', [CompletionResultType]::ParameterName, 'Download the body to a file instead of printing it')
[CompletionResult]::new('--download', '--download', [CompletionResultType]::ParameterName, 'Download the body to a file instead of printing it')
[CompletionResult]::new('-c', '-c', [CompletionResultType]::ParameterName, 'Resume an interrupted download. Requires --download and --output')
@ -111,6 +113,7 @@ Register-ArgumentCompleter -Native -CommandName 'xh' -ScriptBlock {
[CompletionResult]::new('--no-history-print', '--no-history-print', [CompletionResultType]::ParameterName, 'no-history-print')
[CompletionResult]::new('--no-quiet', '--no-quiet', [CompletionResultType]::ParameterName, 'no-quiet')
[CompletionResult]::new('--no-stream', '--no-stream', [CompletionResultType]::ParameterName, 'no-stream')
[CompletionResult]::new('--no-compress', '--no-compress', [CompletionResultType]::ParameterName, 'no-compress')
[CompletionResult]::new('--no-output', '--no-output', [CompletionResultType]::ParameterName, 'no-output')
[CompletionResult]::new('--no-download', '--no-download', [CompletionResultType]::ParameterName, 'no-download')
[CompletionResult]::new('--no-continue', '--no-continue', [CompletionResultType]::ParameterName, 'no-continue')

2
completions/xh.bash
View File

@ -19,7 +19,7 @@ _xh() {
case "${cmd}" in
xh)
opts="-j -f -s -p -h -b -m -v -P -q -S -o -d -c -A -a -F -4 -6 -I -V --json --form --multipart --raw --pretty --format-options --style --response-charset --response-mime --print --headers --body --meta --verbose --debug --all --history-print --quiet --stream --output --download --continue --session --session-read-only --auth-type --auth --bearer --ignore-netrc --offline --check-status --follow --max-redirects --timeout --proxy --verify --cert --cert-key --ssl --native-tls --default-scheme --https --http-version --resolve --interface --ipv4 --ipv6 --ignore-stdin --curl --curl-long --generate --help --no-json --no-form --no-multipart --no-raw --no-pretty --no-format-options --no-style --no-response-charset --no-response-mime --no-print --no-headers --no-body --no-meta --no-verbose --no-debug --no-all --no-history-print --no-quiet --no-stream --no-output --no-download --no-continue --no-session --no-session-read-only --no-auth-type --no-auth --no-bearer --no-ignore-netrc --no-offline --no-check-status --no-follow --no-max-redirects --no-timeout --no-proxy --no-verify --no-cert --no-cert-key --no-ssl --no-native-tls --no-default-scheme --no-https --no-http-version --no-resolve --no-interface --no-ipv4 --no-ipv6 --no-ignore-stdin --no-curl --no-curl-long --no-generate --no-help --version [[METHOD] URL] [REQUEST_ITEM]..."
opts="-j -f -s -p -h -b -m -v -P -q -S -x -o -d -c -A -a -F -4 -6 -I -V --json --form --multipart --raw --pretty --format-options --style --response-charset --response-mime --print --headers --body --meta --verbose --debug --all --history-print --quiet --stream --compress --output --download --continue --session --session-read-only --auth-type --auth --bearer --ignore-netrc --offline --check-status --follow --max-redirects --timeout --proxy --verify --cert --cert-key --ssl --native-tls --default-scheme --https --http-version --resolve --interface --ipv4 --ipv6 --ignore-stdin --curl --curl-long --generate --help --no-json --no-form --no-multipart --no-raw --no-pretty --no-format-options --no-style --no-response-charset --no-response-mime --no-print --no-headers --no-body --no-meta --no-verbose --no-debug --no-all --no-history-print --no-quiet --no-stream --no-compress --no-output --no-download --no-continue --no-session --no-session-read-only --no-auth-type --no-auth --no-bearer --no-ignore-netrc --no-offline --no-check-status --no-follow --no-max-redirects --no-timeout --no-proxy --no-verify --no-cert --no-cert-key --no-ssl --no-native-tls --no-default-scheme --no-https --no-http-version --no-resolve --no-interface --no-ipv4 --no-ipv6 --no-ignore-stdin --no-curl --no-curl-long --no-generate --no-help --version <[METHOD] URL> [REQUEST_ITEM]..."
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0

3
completions/xh.elv
View File

@ -69,6 +69,8 @@ set edit:completion:arg-completer[xh] = {|@words|
cand --quiet 'Do not print to stdout or stderr'
cand -S 'Always stream the response body'
cand --stream 'Always stream the response body'
cand -x 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate'
cand --compress 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate'
cand -d 'Download the body to a file instead of printing it'
cand --download 'Download the body to a file instead of printing it'
cand -c 'Resume an interrupted download. Requires --download and --output'
@ -108,6 +110,7 @@ set edit:completion:arg-completer[xh] = {|@words|
cand --no-history-print 'no-history-print'
cand --no-quiet 'no-quiet'
cand --no-stream 'no-stream'
cand --no-compress 'no-compress'
cand --no-output 'no-output'
cand --no-download 'no-download'
cand --no-continue 'no-continue'

2
completions/xh.fish
View File

@ -35,6 +35,7 @@ complete -c xh -l debug -d 'Print full error stack traces and debug log messages
complete -c xh -l all -d 'Show any intermediary requests/responses while following redirects with --follow'
complete -c xh -s q -l quiet -d 'Do not print to stdout or stderr'
complete -c xh -s S -l stream -d 'Always stream the response body'
complete -c xh -s x -l compress -d 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate'
complete -c xh -s d -l download -d 'Download the body to a file instead of printing it'
complete -c xh -s c -l continue -d 'Resume an interrupted download. Requires --download and --output'
complete -c xh -l ignore-netrc -d 'Do not use credentials from .netrc'
@ -68,6 +69,7 @@ complete -c xh -l no-all
complete -c xh -l no-history-print
complete -c xh -l no-quiet
complete -c xh -l no-stream
complete -c xh -l no-compress
complete -c xh -l no-output
complete -c xh -l no-download
complete -c xh -l no-continue

4
completions/xh.nu
View File

@ -45,6 +45,7 @@ module completions {
--history-print(-P): string # The same as --print but applies only to intermediary requests/responses
--quiet(-q) # Do not print to stdout or stderr
--stream(-S) # Always stream the response body
--compress(-x) # Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate
--output(-o): string # Save output to FILE instead of stdout
--download(-d) # Download the body to a file instead of printing it
--continue(-c) # Resume an interrupted download. Requires --download and --output
@ -77,7 +78,7 @@ module completions {
--curl-long # Use the long versions of curl's flags
--generate: string@"nu-complete xh generate" # Generate shell completions or man pages
--help # Print help
raw_method_or_url?: string # The request URL, preceded by an optional HTTP method
raw_method_or_url: string # The request URL, preceded by an optional HTTP method
...raw_rest_args: string # Optional key-value pairs to be included in the request.
--no-json
--no-form
@ -98,6 +99,7 @@ module completions {
--no-history-print
--no-quiet
--no-stream
--no-compress
--no-output
--no-download
--no-continue

19
doc/xh.1
View File

@ -1,4 +1,4 @@
.TH XH 1 2025-01-04 0.23.1 "User Commands"
.TH XH 1 2025-02-04 0.23.1 "User Commands"
.SH NAME
xh \- Friendly and fast tool for sending HTTP requests
@ -188,6 +188,11 @@ Using quiet twice i.e. \-qq will suppress warnings as well.
\fB\-S\fR, \fB\-\-stream\fR
Always stream the response body.
.TP 4
\fB\-x\fR, \fB\-\-compress\fR
Content compressed (encoded) with Deflate algorithm. The Content\-Encoding header is set to deflate.
Compression is skipped if it appears that compression ratio is negative. Compression can be forced by repeating this option. Note: Compression cannot be forced if the Content\-Encoding request header is present.
.TP 4
\fB\-o\fR, \fB\-\-output\fR=\fIFILE\fR
Save output to FILE instead of stdout.
.TP 4
@ -321,9 +326,17 @@ For translating the other way, try https://curl2httpie.online/.
Use the long versions of curl's flags.
.TP 4
\fB\-\-generate\fR=\fIKIND\fR
Generate shell completions or man pages.
Generate shell completions or man pages. Possible values are:
[possible values: complete\-bash, complete\-elvish, complete\-fish, complete\-nushell, complete\-powershell, complete\-zsh, man]
complete\-bash
complete\-elvish
complete\-fish
complete\-nushell
complete\-powershell
complete\-zsh
man
Example: xh \-\-generate=complete\-bash > xh.bash.
.TP 4
\fB\-\-help\fR
Print help.

11
src/cli.rs
View File

@ -60,7 +60,7 @@ pub struct Cli {
/// Like --form, but force a multipart/form-data request even without files.
///
/// Overrides both --json and --form.
#[clap(long, conflicts_with = "raw", overrides_with_all = &["json", "form"])]
#[clap(long, conflicts_with_all = &["raw", "compress"], overrides_with_all = &["json", "form"])]
pub multipart: bool,
/// Pass raw request data without extra processing.
@ -182,6 +182,15 @@ Example: --print=Hb"
#[clap(short = 'S', long = "stream", name = "stream")]
pub stream_raw: bool,
/// Content compressed (encoded) with Deflate algorithm.
/// The Content-Encoding header is set to deflate.
///
/// Compression is skipped if it appears that compression ratio is negative.
/// Compression can be forced by repeating this option.
/// Note: Compression cannot be used if the Content-Encoding request header is present.
#[clap(short = 'x', long = "compress", name = "compress", action = ArgAction::Count)]
pub compress: u8,
#[clap(skip)]
pub stream: Option<bool>,

2
src/download.rs
View File

@ -60,7 +60,7 @@ fn get_file_name(response: &Response, orig_url: &reqwest::Url) -> String {
.or_else(|| from_url(orig_url))
.unwrap_or_else(|| "index".to_string());
let filename = filename.split(std::path::is_separator).last().unwrap();
let filename = filename.split(std::path::is_separator).next_back().unwrap();
let mut filename = filename.trim().trim_start_matches('.').to_string();

25
src/main.rs
View File

@ -19,7 +19,7 @@ mod utils;
use std::env;
use std::fs::File;
use std::io::{self, IsTerminal, Read};
use std::io::{self, IsTerminal, Read, Write as _};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::path::PathBuf;
use std::process;
@ -28,8 +28,10 @@ use std::sync::Arc;
use anyhow::{anyhow, Context, Result};
use cookie_store::{CookieStore, RawCookie};
use flate2::write::ZlibEncoder;
use hyper::header::CONTENT_ENCODING;
use redirect::RedirectFollower;
use reqwest::blocking::Client;
use reqwest::blocking::{Body as ReqwestBody, Client};
use reqwest::header::{
HeaderValue, ACCEPT, ACCEPT_ENCODING, CONNECTION, CONTENT_TYPE, COOKIE, RANGE, USER_AGENT,
};
@ -508,6 +510,25 @@ fn run(args: Cli) -> Result<i32> {
let mut request = request_builder.headers(headers).build()?;
if args.compress >= 1 {
if request.headers().contains_key(CONTENT_ENCODING) {
// HTTPie overrides the original Content-Encoding header in this case
log::warn!("--compress can't be used with a 'Content-Encoding:' header. --compress will be disabled.");
} else if let Some(body) = request.body_mut() {
// TODO: Compress file body (File) without buffering
let body_bytes = body.buffer()?;
let mut encoder = ZlibEncoder::new(Vec::new(), Default::default());
encoder.write_all(body_bytes)?;
let output = encoder.finish()?;
if output.len() < body_bytes.len() || args.compress >= 2 {
*body = ReqwestBody::from(output);
request
.headers_mut()
.insert(CONTENT_ENCODING, HeaderValue::from_static("deflate"));
}
}
}
for header in &headers_to_unset {
request.headers_mut().remove(header);
}

2
src/to_curl.rs
View File

@ -97,6 +97,8 @@ pub fn translate(args: Cli) -> Result<Command> {
// No equivalent
(args.style.is_some(), "-s/--style"),
// No equivalent
(args.compress > 0, "-x/--compress"),
// No equivalent
(args.response_charset.is_some(), "--response-charset"),
// No equivalent
(args.response_mime.is_some(), "--response-mime"),

234
tests/cases/compress_request_body.rs Normal file
View File

@ -0,0 +1,234 @@
use std::{fs::OpenOptions, io::Read as _};
use hyper::header::HeaderValue;
use predicates::str::contains;
use crate::prelude::*;
use std::io::Write;
fn zlib_decode(bytes: Vec<u8>) -> std::io::Result<String> {
let mut z = flate2::read::ZlibDecoder::new(&bytes[..]);
let mut s = String::new();
z.read_to_string(&mut s)?;
Ok(s)
}
fn server() -> server::Server {
server::http(|req| async move {
match req.uri().path() {
"/deflate" => {
assert_eq!(
req.headers().get(hyper::header::CONTENT_ENCODING),
Some(HeaderValue::from_static("deflate")).as_ref()
);
let compressed_body = req.body().await;
let body = zlib_decode(compressed_body).unwrap();
hyper::Response::builder()
.header("date", "N/A")
.header("Content-Type", "text/plain")
.body(body.into())
.unwrap()
}
"/normal" => {
assert_eq!(req.headers().get(hyper::header::CONTENT_ENCODING), None);
let body = req.body_as_string().await;
hyper::Response::builder()
.header("date", "N/A")
.header("Content-Type", "text/plain")
.body(body.into())
.unwrap()
}
_ => panic!("unknown path"),
}
})
}
#[test]
fn compress_request_body_json() {
let server = server();
get_command()
.arg(format!("{}/deflate", server.base_url()))
.args([
&format!("key={}", "1".repeat(1000)),
"-x",
"-j",
"--pretty=none",
])
.assert()
.stdout(indoc::formatdoc! {r#"
HTTP/1.1 200 OK
Date: N/A
Content-Type: text/plain
Content-Length: 1010
{{"key":"{c}"}}
"#, c = "1".repeat(1000),});
}
#[test]
fn compress_request_body_form() {
let server = server();
get_command()
.arg(format!("{}/deflate", server.base_url()))
.args([
&format!("key={}", "1".repeat(1000)),
"-x",
"-x",
"-f",
"--pretty=none",
])
.assert()
.stdout(indoc::formatdoc! {r#"
HTTP/1.1 200 OK
Date: N/A
Content-Type: text/plain
Content-Length: 1004
key={c}
"#, c = "1".repeat(1000),});
}
#[test]
fn skip_compression_when_compression_ratio_is_negative() {
let server = server();
get_command()
.arg(format!("{}/normal", server.base_url()))
.args([&format!("key={}", "1"), "-x", "-f", "--pretty=none"])
.assert()
.stdout(indoc::formatdoc! {r#"
HTTP/1.1 200 OK
Date: N/A
Content-Type: text/plain
Content-Length: 5
key={c}
"#, c = "1"});
}
#[test]
fn test_compress_force_with_negative_ratio() {
let server = server();
get_command()
.arg(format!("{}/deflate", server.base_url()))
.args([&format!("key={}", "1"), "-xx", "-f", "--pretty=none"])
.assert()
.stdout(indoc::formatdoc! {r#"
HTTP/1.1 200 OK
Date: N/A
Content-Type: text/plain
Content-Length: 5
key={c}
"#, c = "1"});
}
#[test]
fn dont_compress_request_body_if_content_encoding_have_value() {
let server = server::http(|req| async move {
assert_eq!(
req.headers().get(hyper::header::CONTENT_ENCODING),
Some(HeaderValue::from_static("identity")).as_ref()
);
let body = req.body_as_string().await;
hyper::Response::builder()
.header("date", "N/A")
.header("Content-Type", "text/plain")
.body(body.into())
.unwrap()
});
get_command()
.arg(format!("{}/", server.base_url()))
.args([
&format!("key={}", "1".repeat(1000)),
"content-encoding:identity",
"-xx",
"-f",
"--pretty=none",
])
.assert()
.stdout(indoc::formatdoc! {r#"
HTTP/1.1 200 OK
Date: N/A
Content-Type: text/plain
Content-Length: 1004
key={c}
"#, c = "1".repeat(1000),})
.stderr(contains( "warning: --compress can't be used with a 'Content-Encoding:' header. --compress will be disabled."))
.success()
;
}
#[test]
fn compress_body_from_file() {
let server = server::http(|req| async move {
assert_eq!(
req.headers().get(hyper::header::CONTENT_ENCODING),
Some(HeaderValue::from_static("deflate")).as_ref()
);
assert_eq!("Hello world\n", zlib_decode(req.body().await).unwrap());
hyper::Response::default()
});
let dir = tempfile::tempdir().unwrap();
let filename = dir.path().join("input.txt");
OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&filename)
.unwrap()
.write_all(b"Hello world\n")
.unwrap();
get_command()
.arg(server.base_url())
.arg("-xx")
.arg(format!("@{}", filename.to_string_lossy()))
.assert()
.success();
}
#[test]
fn compress_body_from_file_unless_compress_rate_less_1() {
let server = server::http(|req| async move {
assert_eq!(req.headers().get(hyper::header::CONTENT_ENCODING), None);
assert_eq!("Hello world\n", req.body_as_string().await);
hyper::Response::default()
});
let dir = tempfile::tempdir().unwrap();
let filename = dir.path().join("input.txt");
OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&filename)
.unwrap()
.write_all(b"Hello world\n")
.unwrap();
get_command()
.arg(server.base_url())
.arg("-x")
.arg(format!("@{}", filename.to_string_lossy()))
.assert()
.success();
}
#[test]
fn test_cannot_combine_compress_with_multipart() {
get_command()
.arg(format!("{}/deflate", ""))
.args(["--multipart", "-x", "a=1"])
.assert()
.failure()
.stderr(contains(
"the argument '--multipart' cannot be used with '--compress...'",
));
}

1
tests/cases/mod.rs
View File

@ -1,2 +1,3 @@
mod compress_request_body;
mod download;
mod logging;

38
tests/cli.rs
View File

@ -19,6 +19,7 @@ use indoc::indoc;
use predicates::function::function;
use predicates::str::contains;
use reqwest::header::HeaderValue;
use serde_json::Value;
use tempfile::{tempdir, NamedTempFile, TempDir};
pub trait RequestExt {
@ -779,6 +780,43 @@ fn successful_digest_auth() {
.stdout(contains("HTTP/1.1 200 OK"));
}
#[cfg(feature = "online-tests")]
#[test]
fn compress_request_body_online() {
get_command()
.arg("https://postman-echo.com/post")
.args(["--body", "-f", &format!("a={}", "1".repeat(1000))])
.assert()
.stdout(function(|body: &str| {
let json: Value = serde_json::from_str(body).unwrap();
assert_eq!(json["json"]["a"], Value::String("1".repeat(1000)));
let length: i32 = json["headers"]["content-length"]
.as_str()
.unwrap()
.parse()
.unwrap();
assert_eq!(length, 1002);
true
}));
get_command()
.arg("https://postman-echo.com/post")
.args(["-x", "--body", "-f", &format!("a={}", "1".repeat(1000))])
.assert()
.stdout(function(|body: &str| {
let json: Value = serde_json::from_str(body).unwrap();
assert_eq!(json["json"]["a"], Value::String("1".repeat(1000)));
let length: i32 = json["headers"]["content-length"]
.as_str()
.unwrap()
.parse()
.unwrap();
assert!(length < 1000);
true
}));
}
#[cfg(feature = "online-tests")]
#[test]
fn unsuccessful_digest_auth() {