From 51e22479485900d3d593547e8cf855bba9a03db8 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 5 Nov 2017 22:45:41 +0800 Subject: [PATCH] Abbreviate some stdout/stderr output in compiletest. This is intended to prevent the spurious OOM error from run-pass/rustc-rust-log.rs, by skipping the output in the middle when the size is over 416 KB, so that the log output will not be overwhelmed. --- src/Cargo.lock | 2 + src/tools/compiletest/Cargo.toml | 4 + src/tools/compiletest/src/main.rs | 4 +- src/tools/compiletest/src/read2.rs | 208 +++++++++++++++++++++++++++ src/tools/compiletest/src/runtest.rs | 91 +++++++++++- 5 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 src/tools/compiletest/src/read2.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index f8418b77f61a..edd0ee8bd5ac 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -347,7 +347,9 @@ dependencies = [ "getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml index f8282cc5f8d9..d4d567e63c01 100644 --- a/src/tools/compiletest/Cargo.toml +++ b/src/tools/compiletest/Cargo.toml @@ -11,3 +11,7 @@ getopts = "0.2" log = "0.3" rustc-serialize = "0.3" libc = "0.2" + +[target.'cfg(windows)'.dependencies] +miow = "0.2" +winapi = "0.2" diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 1701c8a3e43e..9fb6a3f5e075 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -11,10 +11,11 @@ #![crate_name = "compiletest"] #![feature(test)] +#![feature(slice_rotate)] #![deny(warnings)] -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(unix)] extern crate libc; extern crate test; extern crate getopts; @@ -47,6 +48,7 @@ pub mod runtest; pub mod common; pub mod errors; mod raise_fd_limit; +mod read2; fn main() { env_logger::init().unwrap(); diff --git a/src/tools/compiletest/src/read2.rs b/src/tools/compiletest/src/read2.rs new file mode 100644 index 000000000000..1d8816c7db13 --- /dev/null +++ b/src/tools/compiletest/src/read2.rs @@ -0,0 +1,208 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs` +// Consider unify the read2() in libstd, cargo and this to prevent further code duplication. + +pub use self::imp::read2; + +#[cfg(not(any(unix, windows)))] +mod imp { + use std::io::{self, Read}; + use std::process::{ChildStdout, ChildStderr}; + + pub fn read2(out_pipe: ChildStdout, + err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec, bool)) -> io::Result<()> { + let mut buffer = Vec::new(); + out_pipe.read_to_end(&mut buffer)?; + data(true, &mut buffer, true); + buffer.clear(); + err_pipe.read_to_end(&mut buffer)?; + data(false, &mut buffer, true); + Ok(()) + } +} + +#[cfg(unix)] +mod imp { + use std::io::prelude::*; + use std::io; + use std::mem; + use std::os::unix::prelude::*; + use std::process::{ChildStdout, ChildStderr}; + use libc; + + pub fn read2(mut out_pipe: ChildStdout, + mut err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec, bool)) -> io::Result<()> { + unsafe { + libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK); + libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK); + } + + let mut out_done = false; + let mut err_done = false; + let mut out = Vec::new(); + let mut err = Vec::new(); + + let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() }; + fds[0].fd = out_pipe.as_raw_fd(); + fds[0].events = libc::POLLIN; + fds[1].fd = err_pipe.as_raw_fd(); + fds[1].events = libc::POLLIN; + loop { + // wait for either pipe to become readable using `select` + let r = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) }; + if r == -1 { + let err = io::Error::last_os_error(); + if err.kind() == io::ErrorKind::Interrupted { + continue + } + return Err(err) + } + + // Read as much as we can from each pipe, ignoring EWOULDBLOCK or + // EAGAIN. If we hit EOF, then this will happen because the underlying + // reader will return Ok(0), in which case we'll see `Ok` ourselves. In + // this case we flip the other fd back into blocking mode and read + // whatever's leftover on that file descriptor. + let handle = |res: io::Result<_>| { + match res { + Ok(_) => Ok(true), + Err(e) => { + if e.kind() == io::ErrorKind::WouldBlock { + Ok(false) + } else { + Err(e) + } + } + } + }; + if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? { + out_done = true; + } + data(true, &mut out, out_done); + if !err_done && fds[1].revents != 0 && handle(err_pipe.read_to_end(&mut err))? { + err_done = true; + } + data(false, &mut err, err_done); + + if out_done && err_done { + return Ok(()) + } + } + } +} + +#[cfg(windows)] +mod imp { + extern crate miow; + extern crate winapi; + + use std::io; + use std::os::windows::prelude::*; + use std::process::{ChildStdout, ChildStderr}; + use std::slice; + + use self::miow::iocp::{CompletionPort, CompletionStatus}; + use self::miow::pipe::NamedPipe; + use self::miow::Overlapped; + use self::winapi::ERROR_BROKEN_PIPE; + + struct Pipe<'a> { + dst: &'a mut Vec, + overlapped: Overlapped, + pipe: NamedPipe, + done: bool, + } + + pub fn read2(out_pipe: ChildStdout, + err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec, bool)) -> io::Result<()> { + let mut out = Vec::new(); + let mut err = Vec::new(); + + let port = CompletionPort::new(1)?; + port.add_handle(0, &out_pipe)?; + port.add_handle(1, &err_pipe)?; + + unsafe { + let mut out_pipe = Pipe::new(out_pipe, &mut out); + let mut err_pipe = Pipe::new(err_pipe, &mut err); + + out_pipe.read()?; + err_pipe.read()?; + + let mut status = [CompletionStatus::zero(), CompletionStatus::zero()]; + + while !out_pipe.done || !err_pipe.done { + for status in port.get_many(&mut status, None)? { + if status.token() == 0 { + out_pipe.complete(status); + data(true, out_pipe.dst, out_pipe.done); + out_pipe.read()?; + } else { + err_pipe.complete(status); + data(false, err_pipe.dst, err_pipe.done); + err_pipe.read()?; + } + } + } + + Ok(()) + } + } + + impl<'a> Pipe<'a> { + unsafe fn new(p: P, dst: &'a mut Vec) -> Pipe<'a> { + Pipe { + dst: dst, + pipe: NamedPipe::from_raw_handle(p.into_raw_handle()), + overlapped: Overlapped::zero(), + done: false, + } + } + + unsafe fn read(&mut self) -> io::Result<()> { + let dst = slice_to_end(self.dst); + match self.pipe.read_overlapped(dst, self.overlapped.raw()) { + Ok(_) => Ok(()), + Err(e) => { + if e.raw_os_error() == Some(ERROR_BROKEN_PIPE as i32) { + self.done = true; + Ok(()) + } else { + Err(e) + } + } + } + } + + unsafe fn complete(&mut self, status: &CompletionStatus) { + let prev = self.dst.len(); + self.dst.set_len(prev + status.bytes_transferred() as usize); + if status.bytes_transferred() == 0 { + self.done = true; + } + } + } + + unsafe fn slice_to_end(v: &mut Vec) -> &mut [u8] { + if v.capacity() == 0 { + v.reserve(16); + } + if v.capacity() == v.len() { + v.reserve(1); + } + slice::from_raw_parts_mut(v.as_mut_ptr().offset(v.len() as isize), + v.capacity() - v.len()) + } +} diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index de96aa9cdeb7..0983ce9d09a0 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -29,7 +29,7 @@ use std::fmt; use std::io::prelude::*; use std::io::{self, BufReader}; use std::path::{Path, PathBuf}; -use std::process::{Command, Output, ExitStatus, Stdio}; +use std::process::{Command, Output, ExitStatus, Stdio, Child}; use std::str; use extract_gdb_version; @@ -1344,12 +1344,14 @@ actual:\n\ if let Some(input) = input { child.stdin.as_mut().unwrap().write_all(input.as_bytes()).unwrap(); } - let Output { status, stdout, stderr } = child.wait_with_output().unwrap(); + + let Output { status, stdout, stderr } = read2_abbreviated(child) + .expect("failed to read output"); let result = ProcRes { status, - stdout: String::from_utf8(stdout).unwrap(), - stderr: String::from_utf8(stderr).unwrap(), + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), cmdline, }; @@ -1634,7 +1636,9 @@ actual:\n\ cmd.arg("-a").arg("-u"); cmd.arg(filename); cmd.arg("-nobanner"); - let output = match cmd.output() { + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let output = match cmd.spawn().and_then(read2_abbreviated) { Ok(output) => output, Err(_) => return, }; @@ -2094,6 +2098,8 @@ actual:\n\ let mut cmd = Command::new(make); cmd.current_dir(&self.testpaths.file) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .env("TARGET", &self.config.target) .env("PYTHON", &self.config.docck_python) .env("S", src_root) @@ -2142,7 +2148,7 @@ actual:\n\ } } - let output = cmd.output().expect("failed to spawn `make`"); + let output = cmd.spawn().and_then(read2_abbreviated).expect("failed to spawn `make`"); if !output.status.success() { let res = ProcRes { status: output.status, @@ -2534,3 +2540,76 @@ fn nocomment_mir_line(line: &str) -> &str { line } } + +fn read2_abbreviated(mut child: Child) -> io::Result { + use std::mem::replace; + use read2::read2; + + const HEAD_LEN: usize = 160 * 1024; + const TAIL_LEN: usize = 256 * 1024; + + enum ProcOutput { + Full(Vec), + Abbreviated { + head: Vec, + skipped: usize, + tail: Box<[u8]>, + } + } + + impl ProcOutput { + fn extend(&mut self, data: &[u8]) { + let new_self = match *self { + ProcOutput::Full(ref mut bytes) => { + bytes.extend_from_slice(data); + let new_len = bytes.len(); + if new_len <= HEAD_LEN + TAIL_LEN { + return; + } + let tail = bytes.split_off(new_len - TAIL_LEN).into_boxed_slice(); + let head = replace(bytes, Vec::new()); + let skipped = new_len - HEAD_LEN - TAIL_LEN; + ProcOutput::Abbreviated { head, skipped, tail } + } + ProcOutput::Abbreviated { ref mut skipped, ref mut tail, .. } => { + *skipped += data.len(); + if data.len() <= TAIL_LEN { + tail[..data.len()].copy_from_slice(data); + tail.rotate(data.len()); + } else { + tail.copy_from_slice(&data[(data.len() - TAIL_LEN)..]); + } + return; + } + }; + *self = new_self; + } + + fn into_bytes(self) -> Vec { + match self { + ProcOutput::Full(bytes) => bytes, + ProcOutput::Abbreviated { mut head, skipped, tail } => { + write!(&mut head, "\n\n<<<<<< SKIPPED {} BYTES >>>>>>\n\n", skipped).unwrap(); + head.extend_from_slice(&tail); + head + } + } + } + } + + let mut stdout = ProcOutput::Full(Vec::new()); + let mut stderr = ProcOutput::Full(Vec::new()); + + drop(child.stdin.take()); + read2(child.stdout.take().unwrap(), child.stderr.take().unwrap(), &mut |is_stdout, data, _| { + if is_stdout { &mut stdout } else { &mut stderr }.extend(data); + data.clear(); + })?; + let status = child.wait()?; + + Ok(Output { + status, + stdout: stdout.into_bytes(), + stderr: stderr.into_bytes(), + }) +} \ No newline at end of file