Rollup merge of #37524 - alexcrichton:vendor, r=brson

Vendor all rustbuild dependencies in this repo

This commit vendors all crates.io dependencies into the rust-lang/rust repository using the `cargo-vendor` tool. This is done in an effort to make rustbuild distro-ready by ensuring that our source tarballs are self-contained units which don't need extraneous network downloads.

A new `src/vendor` directory is created with all vendored crates, and Cargo, when using rustbuild, is configured to use this directory. Over time we can deduplicate this directory with the actual src tree (e.g. src/librustc_serialize, src/liblibc, src/libgetopts, ...). For now though that's left to a separate commit.

src/bootstrap/Cargo.toml

@@ -27,10 +27,6 @@ num_cpus = "0.2"
 toml = "0.1"
 getopts = "0.2"
 rustc-serialize = "0.3"
-gcc = "0.3.36"
+gcc = "0.3.38"
 libc = "0.2"
 md5 = "0.1"
-
-[target.'cfg(windows)'.dependencies]
-winapi = "0.2"
-kernel32-sys = "0.2"

src/bootstrap/bootstrap.py

@@ -259,9 +259,11 @@ class RustBuild(object):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
-        self.run([self.cargo(), "build", "--manifest-path",
-                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
-                 env)
+        args = [self.cargo(), "build", "--manifest-path",
+                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_vendored_sources:
+            args.append("--frozen")
+        self.run(args, env)
 
     def run(self, args, env):
         proc = subprocess.Popen(args, env=env)
@@ -400,6 +402,25 @@ def main():
     except:
         pass
 
+    rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
+                              'CFG_ENABLE_VENDOR' in rb.config_mk
+
+    if rb.use_vendored_sources:
+        if not os.path.exists('.cargo'):
+            os.makedirs('.cargo')
+        f = open('.cargo/config','w')
+        f.write("""
+            [source.crates-io]
+            replace-with = 'vendored-sources'
+            registry = 'https://example.com'
+
+            [source.vendored-sources]
+            directory = '{}/src/vendor'
+        """.format(rb.rust_root))
+        f.close()
+    else:
+        if os.path.exists('.cargo'):
+            shutil.rmtree('.cargo')
     data = stage0_data(rb.rust_root)
     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

src/bootstrap/config.rs

@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub vendor: bool,
     pub target_config: HashMap<String, Target>,
 
     // llvm codegen options
@@ -126,6 +127,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    vendor: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -234,6 +236,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);
@@ -347,6 +350,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("VENDOR", self.vendor),
             }
 
             match key {

src/bootstrap/config.toml.example

@@ -82,6 +82,9 @@
 # The path to (or name of) the GDB executable to use
 #gdb = "gdb"
 
+# Indicate whether the vendored sources are used for Rust dependencies or not
+#vendor = false
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================

src/bootstrap/job.rs

@@ -37,15 +37,82 @@
 //! Note that this module has a #[cfg(windows)] above it as none of this logic
 //! is required on Unix.
 
-extern crate kernel32;
-extern crate winapi;
+#![allow(bad_style, dead_code)]
 
 use std::env;
 use std::io;
 use std::mem;
 
-use self::winapi::*;
-use self::kernel32::*;
+type HANDLE = *mut u8;
+type BOOL = i32;
+type DWORD = u32;
+type LPHANDLE = *mut HANDLE;
+type LPVOID = *mut u8;
+type JOBOBJECTINFOCLASS = i32;
+type SIZE_T = usize;
+type LARGE_INTEGER = i64;
+type ULONG_PTR = usize;
+type ULONGLONG = u64;
+
+const FALSE: BOOL = 0;
+const DUPLICATE_SAME_ACCESS: DWORD = 0x2;
+const PROCESS_DUP_HANDLE: DWORD = 0x40;
+const JobObjectExtendedLimitInformation: JOBOBJECTINFOCLASS = 9;
+const JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: DWORD = 0x2000;
+
+extern "system" {
+    fn CreateJobObjectW(lpJobAttributes: *mut u8, lpName: *const u8) -> HANDLE;
+    fn CloseHandle(hObject: HANDLE) -> BOOL;
+    fn GetCurrentProcess() -> HANDLE;
+    fn OpenProcess(dwDesiredAccess: DWORD,
+                   bInheritHandle: BOOL,
+                   dwProcessId: DWORD) -> HANDLE;
+    fn DuplicateHandle(hSourceProcessHandle: HANDLE,
+                       hSourceHandle: HANDLE,
+                       hTargetProcessHandle: HANDLE,
+                       lpTargetHandle: LPHANDLE,
+                       dwDesiredAccess: DWORD,
+                       bInheritHandle: BOOL,
+                       dwOptions: DWORD) -> BOOL;
+    fn AssignProcessToJobObject(hJob: HANDLE, hProcess: HANDLE) -> BOOL;
+    fn SetInformationJobObject(hJob: HANDLE,
+                               JobObjectInformationClass: JOBOBJECTINFOCLASS,
+                               lpJobObjectInformation: LPVOID,
+                               cbJobObjectInformationLength: DWORD) -> BOOL;
+}
+
+#[repr(C)]
+struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION {
+    BasicLimitInformation: JOBOBJECT_BASIC_LIMIT_INFORMATION,
+    IoInfo: IO_COUNTERS,
+    ProcessMemoryLimit: SIZE_T,
+    JobMemoryLimit: SIZE_T,
+    PeakProcessMemoryUsed: SIZE_T,
+    PeakJobMemoryUsed: SIZE_T,
+}
+
+#[repr(C)]
+struct IO_COUNTERS {
+    ReadOperationCount: ULONGLONG,
+    WriteOperationCount: ULONGLONG,
+    OtherOperationCount: ULONGLONG,
+    ReadTransferCount: ULONGLONG,
+    WriteTransferCount: ULONGLONG,
+    OtherTransferCount: ULONGLONG,
+}
+
+#[repr(C)]
+struct JOBOBJECT_BASIC_LIMIT_INFORMATION {
+    PerProcessUserTimeLimit: LARGE_INTEGER,
+    PerJobUserTimeLimit: LARGE_INTEGER,
+    LimitFlags: DWORD,
+    MinimumWorkingsetSize: SIZE_T,
+    MaximumWorkingsetSize: SIZE_T,
+    ActiveProcessLimit: DWORD,
+    Affinity: ULONG_PTR,
+    PriorityClass: DWORD,
+    SchedulingClass: DWORD,
+}
 
 pub unsafe fn setup() {
     // Create a new job object for us to use

src/bootstrap/lib.rs

@@ -460,6 +460,9 @@ impl Build {
         if self.config.rust_optimize {
             cargo.arg("--release");
         }
+        if self.config.vendor {
+            cargo.arg("--frozen");
+        }
         return cargo
     }