From cfb69ad256fc7e16d271d755339345fbfa892b9d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 7 Jul 2015 13:55:36 -0700 Subject: [PATCH] Rewrite dependency installation in Python This commit aims to have the end goal of adding AppVeyor CI support to this repo, and along the way it ended up meaning that the dependency installation bits were rewritten in Python. This has a number of benefits: * Python is more portable than shell * Python is more readable than shell * curl is no longer required on Windows (powershell is used for downloads) There are also a few minor updates made as part of this commit as well: * The README has been updated in how to build Cargo * We now use `sudo: false` on Travis for faster builds. This is done by specifying packages to install instead of installing them ourselves. * pkg-config is no longer listed as a required program --- .travis.install.deps.sh | 76 ++--------------------------------------- .travis.yml | 22 ++++++++---- README.md | 40 ++++++++++------------ appveyor.yml | 19 +++++++++++ configure | 6 ++-- src/etc/dl-snapshot.py | 25 +++----------- src/etc/download.py | 38 +++++++++++++++++++++ src/etc/install-deps.py | 73 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 174 insertions(+), 125 deletions(-) create mode 100644 appveyor.yml create mode 100644 src/etc/download.py create mode 100644 src/etc/install-deps.py diff --git a/.travis.install.deps.sh b/.travis.install.deps.sh index 8e1eb0008..387cf1dd0 100755 --- a/.travis.install.deps.sh +++ b/.travis.install.deps.sh @@ -1,75 +1,5 @@ -#!/bin/bash +#!/bin/sh -set -x -set -e +set -ex -if [ "${TRAVIS_OS_NAME}" = "osx" ] || [ "${PLATFORM}" = "mac" ] || [ "`uname`" = "Darwin" ]; then - target=apple-darwin -elif [ "${OS}" = "Windows_NT" ] || [ "${PLATFORM}" = "win" ]; then - windows=1 -else - target=unknown-linux-gnu -fi - -if [ "${TRAVIS}" = "true" ] && [ "${target}" = "unknown-linux-gnu" ]; then - # Install a 32-bit compiler for linux - sudo apt-get update - if [ "${BITS}" = "32" ]; then - sudo apt-get install libssl-dev:i386 - fi - sudo apt-get install g++-multilib lib32stdc++6 -fi - -url=https://static.rust-lang.org/dist/`cat src/rustversion.txt` - -# On unix hosts install both 32 and 64-bit libraries, but respect BITS to -# determine what arch should be used by default. On windows we don't use 32/64 -# libraries, but instead we install msvc as an alternate architecture. -if [ -z "${windows}" ]; then - if [ "${BITS}" = "32" ]; then - cargo_extra=x86_64-$target - cargo_host=i686-$target - else - cargo_extra=i686-$target - cargo_host=x86_64-$target - fi - libdir=lib -else - if [ "${BITS}" = "32" ]; then - cargo_host=i686-pc-windows-gnu - elif [ "${MSVC}" = "1" ]; then - cargo_host=x86_64-pc-windows-msvc - else - cargo_host=x86_64-pc-windows-gnu - cargo_extra=x86_64-pc-windows-msvc - fi - libdir=bin -fi - -rm -rf rustc *.tar.gz -curl -O $url/rustc-nightly-$cargo_host.tar.gz -tar xfz rustc-nightly-$cargo_host.tar.gz - -if [ ! -z "${cargo_extra}" ]; then - curl -O $url/rustc-nightly-$cargo_extra.tar.gz - tar xfz rustc-nightly-$cargo_extra.tar.gz - - cp -r rustc-nightly-$cargo_extra/rustc/$libdir/rustlib/$cargo_extra \ - rustc-nightly-$cargo_host/rustc/$libdir/rustlib - cp -r rustc-nightly-$cargo_extra/rustc/$libdir/rustlib/$cargo_extra/bin \ - rustc-nightly-$cargo_host/rustc/$libdir/rustlib/$cargo_host - (cd rustc-nightly-$cargo_host && \ - find rustc/$libdir/rustlib/$cargo_extra -type f | \ - sed 's/^rustc\//file:/' >> rustc/manifest.in) - (cd rustc-nightly-$cargo_host && \ - find rustc/$libdir/rustlib/$cargo_host/bin -type f | \ - sed 's/^rustc\//file:/' >> rustc/manifest.in) - rm -rf rustc-nightly-$cargo_extra - rm -f rustc-nightly-$cargo_extra.tar.gz -fi - -./rustc-nightly-$cargo_host/install.sh --prefix=rustc -rm -rf rustc-nightly-$cargo_host -rm -f rustc-nightly-$cargo_host.tar.gz - -set +x +python src/etc/install-deps.py diff --git a/.travis.yml b/.travis.yml index 1f0ceafae..2dd3a38e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,27 @@ language: rust +sudo: false install: - - sh ./.travis.install.deps.sh + - python src/etc/install-deps.py script: - - ./configure --local-rust-root=`pwd`/rustc + - ./configure --local-rust-root=`pwd`/rustc --prefix=$HOME/cargo-install - make - make test - make distcheck - make doc - - sudo make install - - sudo make uninstall + - make install + - make uninstall after_success: | [ $TRAVIS_BRANCH = master ] && [ $TRAVIS_PULL_REQUEST = false ] && [ $(uname -s) = Linux ] && - sudo pip install ghp-import && - ghp-import -n target/doc && - git push -f https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages + pip install ghp-import --user $USER && + $HOME/.local/bin/ghp-import -n target/doc && + git push -qf https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages env: global: - secure: scGpeetUfba5RWyuS4yt10bPoFAI9wpHEReIFqEx7eH5vr2Anajk6+70jW6GdrWVdUvdINiArlQ3An2DeB9vEUWcBjw8WvuPtOH0tDMoSsuVloPlFD8yn1Ac0Bx9getAO5ofxqtoNg+OV4MDVuGabEesqAOWqURNrBC7XK+ntC8= - RUST_TEST_THREADS=1 + os: - linux - osx @@ -27,3 +29,9 @@ os: branches: only: - master + +addons: + apt: + packages: + - g++-multilib + - lib32stdc++6 diff --git a/README.md b/README.md index 25309ba13..20786fcc1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,16 @@ Cargo downloads your Rust project’s dependencies and compiles your project. -Learn more at http://doc.crates.io/. +Learn more at http://doc.crates.io/ -## Installing cargo from nightlies +## Installing Cargo -Cargo has nightlies available for use. The cargo source is not always guaranteed -to compile on rust master as it may lag behind by a day or two. Nightlies, -however, will run regardless of this fact! +Cargo is distributed by default with Rust, so if you've got `rustc` installed +locally you probably also have `cargo` installed locally. -```sh -triple=x86_64-unknown-linux-gnu -curl -O https://static.rust-lang.org/cargo-dist/cargo-nightly-$triple.tar.gz -tar xf cargo-nightly-$triple.tar.gz -./cargo-nightly-$triple/install.sh -``` - -Nightlies are available for the following triples: +If, however, you would like to install Cargo from the nightly binaries that are +generated, you may also do so! Note that these nightlies are not official +binaries, so they are only provided in one format with one installation method. +Each tarball below contains a top-level `install.sh` script to install Cargo. * [`x86_64-unknown-linux-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-unknown-linux-gnu.tar.gz) * [`i686-unknown-linux-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-unknown-linux-gnu.tar.gz) @@ -23,28 +18,27 @@ Nightlies are available for the following triples: * [`i686-apple-darwin`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-apple-darwin.tar.gz) * [`x86_64-pc-windows-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-pc-windows-gnu.tar.gz) * [`i686-pc-windows-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-pc-windows-gnu.tar.gz) +* [`x86_64-pc-windows-msvc`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-pc-windows-msvc.tar.gz) -Note that if you're using the windows snapshot you will need Mingw-w64 installed -as well as MSYS. The installation script needs to be run inside the MSYS shell. +Note that if you're on Windows you will have to run the `install.sh` script from +inside an MSYS shell, likely from a MinGW-64 installation. -## Compiling cargo +## Compiling from Source Cargo requires the following tools and packages to build: * `rustc` * `python` -* `curl` +* `curl` (on Unix) * `cmake` -* `pkg-config` -* OpenSSL headers (`libssl-dev` package on ubuntu) +* OpenSSL headers (only for Unix, this is the `libssl-dev` package on ubuntu) Cargo can then be compiled like many other standard unix-like projects: ```sh git clone https://github.com/rust-lang/cargo cd cargo -git submodule update --init -./.travis.install.deps.sh +python src/etc/install-deps.py ./configure --local-rust-root="$PWD"/rustc make make install @@ -60,7 +54,9 @@ $ ./configure --target=i686-unknown-linux-gnu,x86_64-unknown-linux-gnu ## Adding new subcommands to Cargo -Cargo is designed to be extensible with new subcommands without having to modify Cargo itself. See [the Wiki page][third-party-subcommands] for more details and a list of known community-developed subcommands. +Cargo is designed to be extensible with new subcommands without having to modify +Cargo itself. See [the Wiki page][third-party-subcommands] for more details and +a list of known community-developed subcommands. [third-party-subcommands]: https://github.com/rust-lang/cargo/wiki/Third-party-cargo-subcommands diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..d1071f6ef --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,19 @@ +environment: + matrix: + - MSVC: 1 + BITS: 64 + TARGET: x86_64-pc-windows-msvc + +install: + - python src/etc/install-deps.py + - python src/etc/dl-snapshot.py %TARGET% + - call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64 + - SET PATH=%PATH%;%cd%/rustc/bin + - SET PATH=%PATH%;%cd%/target/snapshot/cargo/bin + - rustc -V + - cargo -V + +build: false + +test_script: + - cargo test diff --git a/configure b/configure index a7f9881f2..e5590d923 100755 --- a/configure +++ b/configure @@ -261,10 +261,10 @@ need_cmd uname need_cmd date need_cmd tr need_cmd sed -need_cmd file need_cmd cmake -need_cmd pkg-config -need_cmd curl +if [ "${OS}" != "Windows_NT" ]; then + need_cmd curl +fi CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/" CFG_BUILD_DIR="$(pwd)/" diff --git a/src/etc/dl-snapshot.py b/src/etc/dl-snapshot.py index a7f8c1907..23f9c5949 100644 --- a/src/etc/dl-snapshot.py +++ b/src/etc/dl-snapshot.py @@ -1,12 +1,9 @@ -import distutils.spawn +import download import hashlib import os -import subprocess -import sys -import tarfile -import shutil -import contextlib import re +import shutil +import sys datere = re.compile('^\d{4}-\d{2}-\d{2}') cksumre = re.compile('^ ([^ ]+) ([^$]+)$') @@ -91,21 +88,9 @@ if os.path.exists(dl_path): exists = True if not exists: - ret = subprocess.call(["curl", "-o", dl_path, url]) - if ret != 0: - raise Exception("failed to fetch url") + download.get(url, dl_path) h = hashlib.sha1(open(dl_path, 'rb').read()).hexdigest() if h != hash: raise Exception("failed to verify the checksum of the snapshot") -with contextlib.closing(tarfile.open(dl_path)) as tar: - for p in tar.getnames(): - name = p.replace("cargo-nightly-" + triple + "/", "", 1) - fp = os.path.join(dst, name) - print("extracting " + p) - tar.extract(p, dst) - tp = os.path.join(dst, p) - if os.path.isdir(tp) and os.path.exists(fp): - continue - shutil.move(tp, fp) -shutil.rmtree(os.path.join(dst, 'cargo-nightly-' + triple)) +download.unpack(dl_path, dst) diff --git a/src/etc/download.py b/src/etc/download.py new file mode 100644 index 000000000..27d4a861f --- /dev/null +++ b/src/etc/download.py @@ -0,0 +1,38 @@ +import contextlib +import os +import shutil +import subprocess +import sys +import tarfile + +def get(url, path): + # see http://serverfault.com/questions/301128/how-to-download + if sys.platform == 'win32': + run(["PowerShell.exe", "/nologo", "-Command", + "(New-Object System.Net.WebClient).DownloadFile('" + url + + "', '" + path + "')"]) + else: + run(["curl", "-o", path, url]) + +def unpack(tarball, dst, quiet=False): + if quiet: + print("extracting " + tarball) + fname = os.path.basename(tarball).replace(".tar.gz", "") + with contextlib.closing(tarfile.open(tarball)) as tar: + for p in tar.getnames(): + name = p.replace(fname + "/", "", 1) + fp = os.path.join(dst, name) + if not quiet: + print("extracting " + p) + tar.extract(p, dst) + tp = os.path.join(dst, p) + if os.path.isdir(tp) and os.path.exists(fp): + continue + shutil.move(tp, fp) + shutil.rmtree(os.path.join(dst, fname)) + +def run(args): + print("running: " + ' '.join(args)) + ret = subprocess.call(args) + if ret != 0: + raise Exception("failed to fetch url: " + url) diff --git a/src/etc/install-deps.py b/src/etc/install-deps.py new file mode 100644 index 000000000..c5be017fe --- /dev/null +++ b/src/etc/install-deps.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import contextlib +import download +import os +import shutil +import sys +import tarfile + +if os.environ.get('BITS') == '32': + host_bits = 'i686' + extra_bits = 'x86_64' +else: + host_bits = 'x86_64' + extra_bits = 'i686' + +extra = None + +# Figure out our target triple +if sys.platform == 'linux' or sys.platform == 'linux2': + host = host_bits + '-unknown-linux-gnu' + extra = extra_bits + '-unknown-linux-gnu' +elif sys.platform == 'darwin': + host = host_bits + '-apple-darwin' + extra = extra_bits + '-apple-darwin' +elif sys.platform == 'win32': + if os.environ.get('MSVC') == '1': + host = host_bits + '-pc-windows-msvc' + else: + host = host_bits + '-pc-windows-gnu' +else: + raise "Unknown platform" + +rust_date = open('src/rustversion.txt').read().strip() +url = 'https://static.rust-lang.org/dist/' + rust_date + +def install_via_tarballs(): + if os.path.isdir("rustc-install"): + shutil.rmtree("rustc-install") + + host_fname = 'rustc-nightly-' + host + '.tar.gz' + download.get(url + '/' + host_fname, host_fname) + download.unpack(host_fname, "rustc-install", quiet=True) + os.remove(host_fname) + + if extra != None: + extra_fname = 'rustc-nightly-' + extra + '.tar.gz' + print("adding target libs for " + extra) + download.get(url + '/' + extra_fname, extra_fname) + manifest = open("rustc-install/rustc/manifest.in", "a") + folder = extra_fname.replace(".tar.gz", "") + with contextlib.closing(tarfile.open(extra_fname)) as tar: + for p in tar.getnames(): + if not "rustc/lib/rustlib/" + extra in p: + continue + name = p.replace(folder + "/", "", 1) + dst = "rustc-install/" + name + f = tar.extract(p, "rustc-install") + tp = os.path.join("rustc-install", p) + if os.path.isdir(tp) and os.path.exists(dst): + continue + shutil.move(tp, dst) + if not os.path.isdir(dst): + manifest.write(p.replace(folder + "/rustc/", "file:") + "\n") + shutil.rmtree("rustc-install/" + folder) + os.remove(extra_fname) + + if os.path.isdir("rustc"): + shutil.rmtree("rustc") + os.rename("rustc-install/rustc", "rustc") + shutil.rmtree("rustc-install") + +install_via_tarballs()