From 6bcdd9ed2cde913e950a545959e42afa3a83a3cd Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:08:53 -0700 Subject: [PATCH 1/7] etc: Delete unused helper script --- src/etc/2014-06-rewrite-bytes-macros.py | 138 ------------------------ 1 file changed, 138 deletions(-) delete mode 100755 src/etc/2014-06-rewrite-bytes-macros.py diff --git a/src/etc/2014-06-rewrite-bytes-macros.py b/src/etc/2014-06-rewrite-bytes-macros.py deleted file mode 100755 index 73ddfcb04cb2..000000000000 --- a/src/etc/2014-06-rewrite-bytes-macros.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -import sys -import subprocess -import re - - -def main(): - if len(sys.argv) <= 1: - print('Usage: %s [ --apply ] filename1.rs filename2.rs ...' - % sys.argv[0]) - elif sys.argv[1] == '--apply': - for filename in sys.argv[2:]: - patch(filename) - else: - for filename in sys.argv[1:]: - diff(filename) - - -def patch(filename): - source = read(filename) - rewritten = rewrite_bytes_macros(source) - if rewritten is not None and rewritten != source: - write(filename, rewritten) - - -def diff(filename): - rewritten = rewrite_bytes_macros(read(filename)) - if rewritten is not None: - p = subprocess.Popen(['diff', '-u', filename, '-'], - stdin=subprocess.PIPE) - p.stdin.write(rewritten) - p.stdin.close() - p.wait() - - -def read(filename): - with open(filename, 'rb') as f: - return f.read() - - -def write(filename, content): - with open(filename, 'wb') as f: - f.write(content) - - -def rewrite_bytes_macros(source): - rewritten, num_occurrences = BYTES_MACRO_RE.subn(rewrite_one_macro, source) - if num_occurrences > 0: - return rewritten - - -BYTES_MACRO_RE = re.compile(br'bytes!\( (?P [^)]* ) \)', re.VERBOSE) - - -def rewrite_one_macro(match): - try: - bytes = parse_bytes(split_args(match.group('args'))) - return b'b"' + b''.join(map(escape, bytes)) + b'"' - except SkipThisRewrite: - print('Skipped: %s' % match.group(0).decode('utf8', 'replace')) - return match.group(0) - - -class SkipThisRewrite(Exception): - pass - - -def split_args(args): - previous = b'' - for arg in args.split(b','): - if previous: - arg = previous + b',' + arg - if arg.count(b'"') % 2 == 0: - yield arg - previous = b'' - else: - previous = arg - if previous: - yield previous - - -def parse_bytes(args): - for arg in args: - arg = arg.strip() - if (arg.startswith(b'"') and arg.endswith(b'"')) or ( - arg.startswith(b"'") and arg.endswith(b"'")): - # Escaped newline means something different in Rust and Python. - if b'\\\n' in arg: - raise SkipThisRewrite - for byte in eval(b'u' + arg).encode('utf8'): - yield ord(byte) - else: - if arg.endswith(b'u8'): - arg = arg[:-2] - # Assume that all Rust integer literals - # are valid Python integer literals - value = int(eval(arg)) - assert value <= 0xFF - yield value - - -def escape(byte): - c = chr(byte) - escaped = { - b'\0': br'\0', - b'\t': br'\t', - b'\n': br'\n', - b'\r': br'\r', - b'\'': b'\\\'', - b'\\': br'\\', - }.get(c) - if escaped is not None: - return escaped - elif b' ' <= c <= b'~': - return chr(byte) - else: - return ('\\x%02X' % byte).encode('ascii') - - -if str is not bytes: - # Python 3.x - ord = lambda x: x - chr = lambda x: bytes([x]) - - -if __name__ == '__main__': - main() From 24bae2e300c8dcf6d32a1b42aee8fccd152e473a Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:12:40 -0700 Subject: [PATCH 2/7] etc: py3 compat for check-summary.py --- src/etc/check-summary.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/etc/check-summary.py b/src/etc/check-summary.py index 917e1970a36c..9312b685c14a 100755 --- a/src/etc/check-summary.py +++ b/src/etc/check-summary.py @@ -34,7 +34,7 @@ if __name__ == '__main__': summaries.append((fname, summary)) def count(t): - return sum(map(lambda (f, s): len(s.get(t, [])), summaries)) + return sum(map(lambda f: len(f[1].get(t, [])), summaries)) logfiles = sys.argv[1:] for files in map(glob.glob, logfiles): @@ -43,15 +43,15 @@ if __name__ == '__main__': failed = count('failed') ignored = count('ignored') measured = count('bench') - print "summary of %d test runs: %d passed; %d failed; %d ignored; %d measured" % \ - (len(logfiles), ok, failed, ignored, measured) - print "" + print("summary of %d test runs: %d passed; %d failed; %d ignored; %d measured" % + (len(logfiles), ok, failed, ignored, measured)) + print("") if failed > 0: - print "failed tests:" + print("failed tests:") for f, s in summaries: failures = s.get('failed', []) if len(failures) > 0: - print " %s:" % (f) + print(" %s:" % (f)) for test in failures: - print " %s" % (test) + print(" %s" % (test)) From 9ecc5a95fc8e05561b66a86d22f23073cfbac5ba Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:22:00 -0700 Subject: [PATCH 3/7] etc: py3 compat for errorck.py --- src/etc/errorck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etc/errorck.py b/src/etc/errorck.py index 7b11504f3cd8..4b1b78da9fde 100644 --- a/src/etc/errorck.py +++ b/src/etc/errorck.py @@ -16,7 +16,7 @@ import os import re if len(sys.argv) < 2: - print "usage: errorck.py " + print("usage: errorck.py ") sys.exit(1) src_dir = sys.argv[1] From 93a02d35071910fb8ae7c824157b6d8c49e36433 Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:31:59 -0700 Subject: [PATCH 4/7] etc: py3 compat for featureck Also rewrite most of the string formatting to be a bit more idiomatic --- src/etc/featureck.py | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/etc/featureck.py b/src/etc/featureck.py index 86fa779cced5..da140e9317d8 100644 --- a/src/etc/featureck.py +++ b/src/etc/featureck.py @@ -21,7 +21,7 @@ import sys, os, re if len(sys.argv) < 2: - print "usage: featurkck.py " + print("usage: featureck.py ") sys.exit(1) src_dir = sys.argv[1] @@ -47,7 +47,7 @@ with open(feature_gate_source, 'r') as f: line = line.replace("(", "").replace("),", "").replace(")", "") parts = line.split(",") if len(parts) != 3: - print "error: unexpected number of components in line: " + original_line + print("error: unexpected number of components in line: " + original_line) sys.exit(1) feature_name = parts[0].strip().replace('"', "") since = parts[1].strip().replace('"', "") @@ -107,9 +107,9 @@ for (dirpath, dirnames, filenames) in os.walk(src_dir): if not mm is None: since = mm.group(1) else: - print "error: misformed stability attribute" - print "line " + str(line_num) + " of " + path + ":" - print line + print("error: misformed stability attribute") + print("line %d of %:" % (line_num, path)) + print(line) errors = True lib_features[feature_name] = feature_name @@ -123,24 +123,24 @@ for (dirpath, dirnames, filenames) in os.walk(src_dir): (expected_since, source_path, source_line_num, source_line) = \ lib_features_and_level.get((feature_name, level)) if since != expected_since: - print "error: mismatch in " + level + " feature '" + feature_name + "'" - print "line " + str(source_line_num) + " of " + source_path + ":" - print source_line - print "line " + str(line_num) + " of " + path + ":" - print line + print("error: mismatch in %s feature '%s'" % (level, feature_name)) + print("line %d of %s:" % (source_line_num, source_path)) + print(source_line) + print("line %d of %s:" % (line_num, path)) + print(line) errors = True # Verify that this lib feature doesn't duplicate a lang feature if feature_name in language_feature_names: - print "error: lib feature '" + feature_name + "' duplicates a lang feature" - print "line " + str(line_num) + " of " + path + ":" - print line + print("error: lib feature '%s' duplicates a lang feature" % (feature_name)) + print("line %d of %s:" % (line_num, path)) + print(line) errors = True else: - print "error: misformed stability attribute" - print "line " + str(line_num) + " of " + path + ":" - print line + print("error: misformed stability attribute") + print("line %d of %s:" % (line_num, path)) + print(line) errors = True # Merge data about both lists @@ -175,7 +175,7 @@ for f in lib_features: is_unstable = lib_features_and_level.get((name, "unstable")) is not None if is_stable and is_unstable: - print "error: feature '" + name + "' is both stable and unstable" + print("error: feature '%s' is both stable and unstable" % (name)) errors = True if is_stable: @@ -192,7 +192,7 @@ merged_stats = { } for name in lib_feature_stats: if language_feature_stats.get(name) is not None: if not name in joint_features: - print "error: feature '" + name + "' is both a lang and lib feature but not whitelisted" + print("error: feature '%s' is both a lang and lib feature but not whitelisted" % (name)) errors = True lang_status = language_feature_stats[name][3] lib_status = lib_feature_stats[name][3] @@ -200,13 +200,13 @@ for name in lib_feature_stats: lib_stable_since = lib_feature_stats[name][4] if lang_status != lib_status and lib_status != "deprecated": - print "error: feature '" + name + "' has lang status " + lang_status + \ - " but lib status " + lib_status + print("error: feature '%s' has lang status %s " + + "but lib status %s" % (name, lang_status, lib_status)) errors = True if lang_stable_since != lib_stable_since: - print "error: feature '" + name + "' has lang stable since " + lang_stable_since + \ - " but lib stable since " + lib_stable_since + print("error: feature '%s' has lang stable since %s " + + "but lib stable since %s" % (name, lang_stable_since, lib_stable_since)) errors = True merged_stats[name] = (name, True, True, lang_status, lang_stable_since) @@ -240,5 +240,5 @@ lines.sort() print for line in lines: - print "* " + line + print("* " + line) print From 4decc408dc1dcac5b79edd41760cb85d3dde5a01 Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:34:29 -0700 Subject: [PATCH 5/7] etc: py3 compat for tidy.py --- src/etc/tidy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/etc/tidy.py b/src/etc/tidy.py index c524fae7f0a4..9f5f919bce8d 100644 --- a/src/etc/tidy.py +++ b/src/etc/tidy.py @@ -81,7 +81,7 @@ check_cr = True check_linelength = True if len(sys.argv) < 2: - print "usage: tidy.py " + print("usage: tidy.py ") sys.exit(1) src_dir = sys.argv[1] @@ -200,10 +200,10 @@ except UnicodeDecodeError as e: print for ext in sorted(file_counts, key=file_counts.get, reverse=True): - print "* linted {} {} files".format(file_counts[ext], ext) -print "* linted {} other files".format(count_other_linted_files) -print "* total lines of code: {}".format(count_lines) -print "* total non-blank lines of code: {}".format(count_non_blank_lines) -print + print("* linted {} {} files".format(file_counts[ext], ext)) +print("* linted {} other files".format(count_other_linted_files)) +print("* total lines of code: {}".format(count_lines)) +print("* total non-blank lines of code: {}".format(count_non_blank_lines)) +print() sys.exit(err) From d1082aa3a1ec7d039b58eba994e32b79db778381 Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Sun, 24 May 2015 05:41:38 -0700 Subject: [PATCH 6/7] etc: work around utf8 text in rust sources on py3 in featureck --- src/etc/featureck.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/etc/featureck.py b/src/etc/featureck.py index da140e9317d8..ecb2e6db466d 100644 --- a/src/etc/featureck.py +++ b/src/etc/featureck.py @@ -78,8 +78,15 @@ for (dirpath, dirnames, filenames) in os.walk(src_dir): if not filename.endswith(".rs"): continue + if sys.version_info.major == 2: + _open = lambda f: open(f, 'r') + elif sys.version_info.major == 3: + _open = lambda f: open(f, 'r', encoding="utf-8") + else: + raise RuntimeError("Unsupported python version: %s" % (repr(sys.version_info))) + path = os.path.join(dirpath, filename) - with open(path, 'r') as f: + with _open(path) as f: line_num = 0 for line in f: line_num += 1 From 96d7400b1a1e2e8f3ac052015684fc3946e2da19 Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Tue, 26 May 2015 12:09:28 -0700 Subject: [PATCH 7/7] etc: use codecs in featureck this asserts that source is valid utf8 on both python3 and python2 --- src/etc/featureck.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/etc/featureck.py b/src/etc/featureck.py index ecb2e6db466d..e82f00f3e7df 100644 --- a/src/etc/featureck.py +++ b/src/etc/featureck.py @@ -18,7 +18,10 @@ # since the same version # * Prints information about features -import sys, os, re +import sys +import os +import re +import codecs if len(sys.argv) < 2: print("usage: featureck.py ") @@ -78,15 +81,8 @@ for (dirpath, dirnames, filenames) in os.walk(src_dir): if not filename.endswith(".rs"): continue - if sys.version_info.major == 2: - _open = lambda f: open(f, 'r') - elif sys.version_info.major == 3: - _open = lambda f: open(f, 'r', encoding="utf-8") - else: - raise RuntimeError("Unsupported python version: %s" % (repr(sys.version_info))) - path = os.path.join(dirpath, filename) - with _open(path) as f: + with codecs.open(filename=path, mode='r', encoding="utf-8") as f: line_num = 0 for line in f: line_num += 1