734 lines
42 KiB
HTML
734 lines
42 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="light sidebar-visible" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>UI tests - Rust Compiler Development Guide</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="A guide to developing the Rust compiler (rustc)">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
|
|
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
|
|
|
|
<!-- Provide site root and default themes to javascript -->
|
|
<script>
|
|
const path_to_root = "../";
|
|
const default_light_theme = "light";
|
|
const default_dark_theme = "navy";
|
|
</script>
|
|
<!-- Start loading toc.js asap -->
|
|
<script src="../toc.js"></script>
|
|
</head>
|
|
<body>
|
|
<div id="body-container">
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
let theme = localStorage.getItem('mdbook-theme');
|
|
let sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
|
let theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
const html = document.documentElement;
|
|
html.classList.remove('light')
|
|
html.classList.add(theme);
|
|
html.classList.add("js");
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
let sidebar = null;
|
|
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<!-- populated by js -->
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
|
<noscript>
|
|
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
|
|
</noscript>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
|
<div class="sidebar-resize-indicator"></div>
|
|
</div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Rust Compiler Development Guide</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
<a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
|
</a>
|
|
<a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/tests/ui.md" title="Suggest an edit" aria-label="Suggest an edit">
|
|
<i id="git-edit-button" class="fa fa-edit"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<h1 id="ui-tests"><a class="header" href="#ui-tests">UI tests</a></h1>
|
|
<ul>
|
|
<li><a href="#introduction">Introduction</a></li>
|
|
<li><a href="#general-structure-of-a-test">General structure of a test</a></li>
|
|
<li><a href="#output-comparison">Output comparison</a>
|
|
<ul>
|
|
<li><a href="#normalization">Normalization</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#error-annotations">Error annotations</a>
|
|
<ul>
|
|
<li><a href="#error-annotation-examples">Error annotation examples</a>
|
|
<ul>
|
|
<li><a href="#positioned-on-error-line">Positioned on error line</a></li>
|
|
<li><a href="#positioned-below-error-line">Positioned below error line</a></li>
|
|
<li><a href="#use-same-error-line-as-defined-on-error-annotation-line-above">Use same error line as defined on error annotation line above</a></li>
|
|
<li><a href="#positioned-above-error-line">Positioned above error line</a></li>
|
|
<li><a href="#error-without-line-information">Error without line information</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#error-pattern"><code>error-pattern</code></a></li>
|
|
<li><a href="#diagnostic-kinds-error-levels">Diagnostic kinds (error levels)</a></li>
|
|
<li><a href="#cfg-revisions"><code>cfg</code> revisions</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#controlling-passfail-expectations">Controlling pass/fail expectations</a></li>
|
|
<li><a href="#known-bugs">Known bugs</a></li>
|
|
<li><a href="#test-organization">Test organization</a></li>
|
|
<li><a href="#rustfix-tests">Rustfix tests</a></li>
|
|
<li><a href="#compare-modes">Compare modes</a></li>
|
|
<li><a href="#rustc_-test-attributes"><code>rustc_*</code> TEST attributes</a></li>
|
|
<li><a href="#ui-test-mode-preset-lint-levels">UI test mode preset lint levels</a></li>
|
|
</ul>
|
|
<p>UI tests are a particular <a href="compiletest.html#test-suites">test suite</a> of
|
|
compiletest.</p>
|
|
<h2 id="introduction"><a class="header" href="#introduction">Introduction</a></h2>
|
|
<p>The tests in <a href="https://github.com/rust-lang/rust/blob/master/tests/ui"><code>tests/ui</code></a> are a collection of general-purpose tests which
|
|
primarily focus on validating the console output of the compiler, but can be
|
|
used for many other purposes. For example, tests can also be configured to <a href="#controlling-passfail-expectations">run
|
|
the resulting program</a> to verify its
|
|
behavior.</p>
|
|
<p>If you need to work with <code>#![no_std]</code> cross-compiling tests, consult the
|
|
<a href="./minicore.html"><code>minicore</code> test auxiliary</a> chapter.</p>
|
|
<h2 id="general-structure-of-a-test"><a class="header" href="#general-structure-of-a-test">General structure of a test</a></h2>
|
|
<p>A test consists of a Rust source file located anywhere in the <code>tests/ui</code>
|
|
directory, but they should be placed in a suitable sub-directory. For example,
|
|
<a href="https://github.com/rust-lang/rust/blob/master/tests/ui/hello.rs"><code>tests/ui/hello.rs</code></a> is a basic hello-world test.</p>
|
|
<p>Compiletest will use <code>rustc</code> to compile the test, and compare the output against
|
|
the expected output which is stored in a <code>.stdout</code> or <code>.stderr</code> file located
|
|
next to the test. See <a href="#output-comparison">Output comparison</a> for more.</p>
|
|
<p>Additionally, errors and warnings should be annotated with comments within the
|
|
source file. See <a href="#error-annotations">Error annotations</a> for more.</p>
|
|
<p>Compiletest <a href="directives.html">directives</a> in the form of special comments prefixed
|
|
with <code>//@</code> control how the test is compiled and what the expected behavior is.</p>
|
|
<p>Tests are expected to fail to compile, since most tests are testing compiler
|
|
errors. You can change that behavior with a directive, see <a href="#controlling-passfail-expectations">Controlling
|
|
pass/fail expectations</a>.</p>
|
|
<p>By default, a test is built as an executable binary. If you need a different
|
|
crate type, you can use the <code>#![crate_type]</code> attribute to set it as needed.</p>
|
|
<h2 id="output-comparison"><a class="header" href="#output-comparison">Output comparison</a></h2>
|
|
<p>UI tests store the expected output from the compiler in <code>.stderr</code> and <code>.stdout</code>
|
|
snapshots next to the test. You normally generate these files with the <code>--bless</code>
|
|
CLI option, and then inspect them manually to verify they contain what you
|
|
expect.</p>
|
|
<p>The output is normalized to ignore unwanted differences, see the
|
|
<a href="#normalization">Normalization</a> section. If the file is missing, then
|
|
compiletest expects the corresponding output to be empty.</p>
|
|
<p>There can be multiple stdout/stderr files. The general form is:</p>
|
|
<pre><code class="language-text">*test-name*`.`*revision*`.`*compare_mode*`.`*extension*
|
|
</code></pre>
|
|
<ul>
|
|
<li><em>test-name</em> cannot contain dots. This is so that the general form of test
|
|
output filenames have a predictable form we can pattern match on in order to
|
|
track stray test output files.</li>
|
|
<li><em>revision</em> is the <a href="#cfg-revisions">revision</a> name. This is not included when
|
|
not using revisions.</li>
|
|
<li><em>compare_mode</em> is the <a href="#compare-modes">compare mode</a>. This will only be
|
|
checked when the given compare mode is active. If the file does not exist,
|
|
then compiletest will check for a file without the compare mode.</li>
|
|
<li><em>extension</em> is the kind of output being checked:
|
|
<ul>
|
|
<li><code>stderr</code> — compiler stderr</li>
|
|
<li><code>stdout</code> — compiler stdout</li>
|
|
<li><code>run.stderr</code> — stderr when running the test</li>
|
|
<li><code>run.stdout</code> — stdout when running the test</li>
|
|
<li><code>64bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a
|
|
64-bit target</li>
|
|
<li><code>32bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a
|
|
32-bit target</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<p>A simple example would be <code>foo.stderr</code> next to a <code>foo.rs</code> test.
|
|
A more complex example would be <code>foo.my-revision.polonius.stderr</code>.</p>
|
|
<p>There are several <a href="directives.html">directives</a> which will change how compiletest
|
|
will check for output files:</p>
|
|
<ul>
|
|
<li><code>stderr-per-bitwidth</code> — checks separate output files based on the target
|
|
pointer width. Consider using the <code>normalize-stderr</code> directive instead (see
|
|
<a href="#normalization">Normalization</a>).</li>
|
|
<li><code>dont-check-compiler-stderr</code> — Ignores stderr from the compiler.</li>
|
|
<li><code>dont-check-compiler-stdout</code> — Ignores stdout from the compiler.</li>
|
|
</ul>
|
|
<p>UI tests run with <code>-Zdeduplicate-diagnostics=no</code> flag which disables rustc's
|
|
built-in diagnostic deduplication mechanism. This means you may see some
|
|
duplicate messages in the output. This helps illuminate situations where
|
|
duplicate diagnostics are being generated.</p>
|
|
<h3 id="normalization"><a class="header" href="#normalization">Normalization</a></h3>
|
|
<p>The compiler output is normalized to eliminate output difference between
|
|
platforms, mainly about filenames.</p>
|
|
<p>Compiletest makes the following replacements on the compiler output:</p>
|
|
<ul>
|
|
<li>The directory where the test is defined is replaced with <code>$DIR</code>. Example:
|
|
<code>/path/to/rust/tests/ui/error-codes</code></li>
|
|
<li>The directory to the standard library source is replaced with <code>$SRC_DIR</code>.
|
|
Example: <code>/path/to/rust/library</code></li>
|
|
<li>Line and column numbers for paths in <code>$SRC_DIR</code> are replaced with <code>LL:COL</code>.
|
|
This helps ensure that changes to the layout of the standard library do not
|
|
cause widespread changes to the <code>.stderr</code> files. Example:
|
|
<code>$SRC_DIR/alloc/src/sync.rs:53:46</code></li>
|
|
<li>The base directory where the test's output goes is replaced with
|
|
<code>$TEST_BUILD_DIR</code>. This only comes up in a few rare circumstances. Example:
|
|
<code>/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui</code></li>
|
|
<li>The real directory to the standard library source is replaced with <code>$SRC_DIR_REAL</code>.</li>
|
|
<li>The real directory to the compiler source is replaced with <code>$COMPILER_DIR_REAL</code>.</li>
|
|
<li>Tabs are replaced with <code>\t</code>.</li>
|
|
<li>Backslashes (<code>\</code>) are converted to forward slashes (<code>/</code>) within paths (using a
|
|
heuristic). This helps normalize differences with Windows-style paths.</li>
|
|
<li>CRLF newlines are converted to LF.</li>
|
|
<li>Error line annotations like <code>//~ ERROR some message</code> are removed.</li>
|
|
<li>Various v0 and legacy symbol hashes are replaced with placeholders like
|
|
<code>[HASH]</code> or <code><SYMBOL_HASH></code>.</li>
|
|
</ul>
|
|
<p>Additionally, the compiler is run with the <code>-Z ui-testing</code> flag which causes
|
|
the compiler itself to apply some changes to the diagnostic output to make it
|
|
more suitable for UI testing.</p>
|
|
<p>For example, it will anonymize line numbers in the output (line numbers
|
|
prefixing each source line are replaced with <code>LL</code>). In extremely rare
|
|
situations, this mode can be disabled with the directive <code>//@ compile-flags: -Z ui-testing=no</code>.</p>
|
|
<p>Note: The line and column numbers for <code>--></code> lines pointing to the test are <em>not</em>
|
|
normalized, and left as-is. This ensures that the compiler continues to point to
|
|
the correct location, and keeps the stderr files readable. Ideally all
|
|
line/column information would be retained, but small changes to the source
|
|
causes large diffs, and more frequent merge conflicts and test errors.</p>
|
|
<p>Sometimes these built-in normalizations are not enough. In such cases, you may
|
|
provide custom normalization rules using <code>normalize-*</code> directives, e.g.</p>
|
|
<pre><code class="language-rust ignore">//@ normalize-stdout: "foo" -> "bar"
|
|
//@ normalize-stderr: "foo" -> "bar"
|
|
//@ normalize-stderr-32bit: "fn\(\) \(32 bits\)" -> "fn\(\) \($$PTR bits\)"
|
|
//@ normalize-stderr-64bit: "fn\(\) \(64 bits\)" -> "fn\(\) \($$PTR bits\)"</code></pre>
|
|
<p>This tells the test, on 32-bit platforms, whenever the compiler writes <code>fn() (32 bits)</code> to stderr, it should be normalized to read <code>fn() ($PTR bits)</code> instead.
|
|
Similar for 64-bit. The replacement is performed by regexes using default regex
|
|
flavor provided by <code>regex</code> crate.</p>
|
|
<p>The corresponding reference file will use the normalized output to test both
|
|
32-bit and 64-bit platforms:</p>
|
|
<pre><code class="language-text">...
|
|
|
|
|
= note: source type: fn() ($PTR bits)
|
|
= note: target type: u16 (16 bits)
|
|
...
|
|
</code></pre>
|
|
<p>Please see <a href="https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.rs"><code>ui/transmute/main.rs</code></a> and <a href="https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.stderr"><code>main.stderr</code></a> for a concrete
|
|
usage example.</p>
|
|
<h2 id="error-annotations"><a class="header" href="#error-annotations">Error annotations</a></h2>
|
|
<p>Error annotations specify the errors that the compiler is expected to emit. They
|
|
are "attached" to the line in source where the error is located.</p>
|
|
<pre><code class="language-rust ignore">fn main() {
|
|
boom //~ ERROR cannot find value `boom` in this scope [E0425]
|
|
}</code></pre>
|
|
<p>Although UI tests have a <code>.stderr</code> file which contains the entire compiler
|
|
output, UI tests require that errors are also annotated within the source. This
|
|
redundancy helps avoid mistakes since the <code>.stderr</code> files are usually
|
|
auto-generated. It also helps to directly see where the error spans are expected
|
|
to point to by looking at one file instead of having to compare the <code>.stderr</code>
|
|
file with the source. Finally, they ensure that no additional unexpected errors
|
|
are generated.</p>
|
|
<p>They have several forms, but generally are a comment with the diagnostic level
|
|
(such as <code>ERROR</code>) and a substring of the expected error output. You don't have
|
|
to write out the entire message, just make sure to include the important part of
|
|
the message to make it self-documenting.</p>
|
|
<p>Most error annotations need to match with the line of the diagnostic. There are
|
|
several ways to match the message with the line (see the examples below):</p>
|
|
<ul>
|
|
<li><code>~</code>: Associates the error level and message with the <em>current</em> line</li>
|
|
<li><code>~^</code>: Associates the error level and message with the <em>previous</em> error
|
|
annotation line. Each caret (<code>^</code>) that you add adds a line to this, so <code>~^^^</code>
|
|
is three lines above the error annotation line.</li>
|
|
<li><code>~|</code>: Associates the error level and message with the <em>same</em> line as the
|
|
<em>previous comment</em>. This is more convenient than using multiple carets when
|
|
there are multiple messages associated with the same line.</li>
|
|
<li><code>~v</code>: Associates the error level and message with the <em>next</em> error
|
|
annotation line. Each symbol (<code>v</code>) that you add adds a line to this, so <code>~vvv</code>
|
|
is three lines below the error annotation line.</li>
|
|
</ul>
|
|
<p>Example:</p>
|
|
<pre><code class="language-rust ignore">let _ = same_line; //~ ERROR undeclared variable
|
|
fn meow(_: [u8]) {}
|
|
//~^ ERROR unsized
|
|
//~| ERROR anonymous parameters</code></pre>
|
|
<p>The space character between <code>//~</code> (or other variants) and the subsequent text is
|
|
negligible (i.e. there is no semantic difference between <code>//~ ERROR</code> and
|
|
<code>//~ERROR</code> although the former is more common in the codebase).</p>
|
|
<p><code>~? <diagnostic kind></code> (example being <code>~? ERROR</code>)
|
|
is used to match diagnostics <em>without</em> line info at all,
|
|
or where the line info is outside the main test file<sup class="footnote-reference" id="fr-main test file-1"><a href="#footnote-main test file">1</a></sup>.
|
|
These annotations can be placed on any line in the test file.</p>
|
|
<h3 id="error-annotation-examples"><a class="header" href="#error-annotation-examples">Error annotation examples</a></h3>
|
|
<p>Here are examples of error annotations on different lines of UI test source.</p>
|
|
<h4 id="positioned-on-error-line"><a class="header" href="#positioned-on-error-line">Positioned on error line</a></h4>
|
|
<p>Use the <code>//~ ERROR</code> idiom:</p>
|
|
<pre><code class="language-rust ignore">fn main() {
|
|
let x = (1, 2, 3);
|
|
match x {
|
|
(_a, _x @ ..) => {} //~ ERROR `_x @` is not allowed in a tuple
|
|
_ => {}
|
|
}
|
|
}</code></pre>
|
|
<h4 id="positioned-below-error-line"><a class="header" href="#positioned-below-error-line">Positioned below error line</a></h4>
|
|
<p>Use the <code>//~^</code> idiom with number of carets in the string to indicate the number
|
|
of lines above. In the example below, the error line is four lines above the
|
|
error annotation line so four carets are included in the annotation.</p>
|
|
<pre><code class="language-rust ignore">fn main() {
|
|
let x = (1, 2, 3);
|
|
match x {
|
|
(_a, _x @ ..) => {} // <- the error is on this line
|
|
_ => {}
|
|
}
|
|
}
|
|
//~^^^^ ERROR `_x @` is not allowed in a tuple</code></pre>
|
|
<h4 id="use-same-error-line-as-defined-on-error-annotation-line-above"><a class="header" href="#use-same-error-line-as-defined-on-error-annotation-line-above">Use same error line as defined on error annotation line above</a></h4>
|
|
<p>Use the <code>//~|</code> idiom to define the same error line as the error annotation
|
|
line above:</p>
|
|
<pre><code class="language-rust ignore">struct Binder(i32, i32, i32);
|
|
|
|
fn main() {
|
|
let x = Binder(1, 2, 3);
|
|
match x {
|
|
Binder(_a, _x @ ..) => {} // <- the error is on this line
|
|
_ => {}
|
|
}
|
|
}
|
|
//~^^^^ ERROR `_x @` is not allowed in a tuple struct
|
|
//~| ERROR this pattern has 1 field, but the corresponding tuple struct has 3 fields [E0023]</code></pre>
|
|
<h4 id="positioned-above-error-line"><a class="header" href="#positioned-above-error-line">Positioned above error line</a></h4>
|
|
<p>Use the <code>//~v</code> idiom with number of v's in the string to indicate the number
|
|
of lines below. This is typically used in lexer or parser tests matching on errors like unclosed
|
|
delimiter or unclosed literal happening at the end of file.</p>
|
|
<pre><code class="language-rust ignore">// ignore-tidy-trailing-newlines
|
|
//~v ERROR this file contains an unclosed delimiter
|
|
fn main((ؼ</code></pre>
|
|
<h4 id="error-without-line-information"><a class="header" href="#error-without-line-information">Error without line information</a></h4>
|
|
<p>Use <code>//~?</code> to match an error without line information.
|
|
<code>//~?</code> is precise and will not match errors if their line information is available.
|
|
It should be preferred to using <code>error-pattern</code>, which is imprecise and non-exhaustive.</p>
|
|
<pre><code class="language-rust ignore">//@ compile-flags: --print yyyy
|
|
|
|
//~? ERROR unknown print request: `yyyy`</code></pre>
|
|
<h3 id="error-pattern"><a class="header" href="#error-pattern"><code>error-pattern</code></a></h3>
|
|
<p>The <code>error-pattern</code> <a href="directives.html">directive</a> can be used for runtime messages, which don't
|
|
have a specific span, or in exceptional cases, for compile time messages.</p>
|
|
<p>Let's think about this test:</p>
|
|
<pre><code class="language-rust ignore">fn main() {
|
|
let a: *const [_] = &[1, 2, 3];
|
|
unsafe {
|
|
let _b = (*a)[3];
|
|
}
|
|
}</code></pre>
|
|
<p>We want to ensure this shows "index out of bounds", but we cannot use the <code>ERROR</code>
|
|
annotation since the runtime error doesn't have any span. Then it's time to use the
|
|
<code>error-pattern</code> directive:</p>
|
|
<pre><code class="language-rust ignore">//@ error-pattern: index out of bounds
|
|
fn main() {
|
|
let a: *const [_] = &[1, 2, 3];
|
|
unsafe {
|
|
let _b = (*a)[3];
|
|
}
|
|
}</code></pre>
|
|
<p>Use of <code>error-pattern</code> is not recommended in general.</p>
|
|
<p>For strict testing of compile time output, try to use the line annotations <code>//~</code> as much as
|
|
possible, including <code>//~?</code> annotations for diagnostics without spans.</p>
|
|
<p>If the compile time output is target dependent or too verbose, use directive
|
|
<code>//@ dont-require-annotations: <diagnostic-kind></code> to make the line annotation checking
|
|
non-exhaustive.
|
|
Some of the compiler messages can stay uncovered by annotations in this mode.</p>
|
|
<p>For checking runtime output, <code>//@ check-run-results</code> may be preferable.</p>
|
|
<p>Only use <code>error-pattern</code> if none of the above works.</p>
|
|
<p>Line annotations <code>//~</code> and <code>error-pattern</code> are compatible and can be used in the same test.</p>
|
|
<h3 id="diagnostic-kinds-error-levels"><a class="header" href="#diagnostic-kinds-error-levels">Diagnostic kinds (error levels)</a></h3>
|
|
<p>The diagnostic kinds that you can have are:</p>
|
|
<ul>
|
|
<li><code>ERROR</code></li>
|
|
<li><code>WARN</code> (or <code>WARNING</code>)</li>
|
|
<li><code>NOTE</code></li>
|
|
<li><code>HELP</code></li>
|
|
<li><code>SUGGESTION</code></li>
|
|
<li><code>RAW</code></li>
|
|
</ul>
|
|
<p>The <code>SUGGESTION</code> kind is used for specifying what the expected replacement text
|
|
should be for a diagnostic suggestion.
|
|
The <code>RAW</code> kind can be used for matching on lines from non-structured output sometimes emitted
|
|
by the compiler instead of or in addition to structured json.</p>
|
|
<p><code>ERROR</code> and <code>WARN</code> kinds are required to be exhaustively covered by line annotations
|
|
<code>//~</code> by default.</p>
|
|
<p>Other kinds only need to be line-annotated if at least one annotation of that kind appears
|
|
in the test file. For example, one <code>//~ NOTE</code> will also require all other <code>//~ NOTE</code>s in the file
|
|
to be written out explicitly.</p>
|
|
<p>Use directive <code>//@ dont-require-annotations</code> to opt out of exhaustive annotations.
|
|
E.g. use <code>//@ dont-require-annotations: NOTE</code> to annotate notes selectively.
|
|
Avoid using this directive for <code>ERROR</code>s and <code>WARN</code>ings, unless there's a serious reason, like
|
|
target-dependent compiler output.</p>
|
|
<p>Some diagnostics are never required to be line-annotated, regardless of their kind or directives,
|
|
for example secondary lines of multiline diagnostics,
|
|
or ubiquitous diagnostics like <code>aborting due to N previous errors</code>.</p>
|
|
<p>UI tests use the <code>-A unused</code> flag by default to ignore all unused warnings, as
|
|
unused warnings are usually not the focus of a test. However, simple code
|
|
samples often have unused warnings. If the test is specifically testing an
|
|
unused warning, just add the appropriate <code>#![warn(unused)]</code> attribute as needed.</p>
|
|
<h3 id="cfg-revisions"><a class="header" href="#cfg-revisions"><code>cfg</code> revisions</a></h3>
|
|
<p>When using <a href="compiletest.html#revisions">revisions</a>, different messages can be
|
|
conditionally checked based on the current revision. This is done by placing the
|
|
revision cfg name in brackets like this:</p>
|
|
<pre><code class="language-rust ignore">//@ edition:2018
|
|
//@ revisions: mir thir
|
|
//@[thir] compile-flags: -Z thir-unsafeck
|
|
|
|
async unsafe fn f() {}
|
|
|
|
async fn g() {
|
|
f(); //~ ERROR call to unsafe function is unsafe
|
|
}
|
|
|
|
fn main() {
|
|
f(); //[mir]~ ERROR call to unsafe function is unsafe
|
|
}</code></pre>
|
|
<p>In this example, the second error message is only emitted in the <code>mir</code> revision.
|
|
The <code>thir</code> revision only emits the first error.</p>
|
|
<p>If the <code>cfg</code> causes the compiler to emit different output, then a test can have
|
|
multiple <code>.stderr</code> files for the different outputs. In the example above, there
|
|
would be a <code>.mir.stderr</code> and <code>.thir.stderr</code> file with the different outputs of
|
|
the different revisions.</p>
|
|
<blockquote>
|
|
<p>Note: cfg revisions also work inside the source code with <code>#[cfg]</code> attributes.</p>
|
|
<p>By convention, the <code>FALSE</code> cfg is used to have an always-false config.</p>
|
|
</blockquote>
|
|
<h2 id="controlling-passfail-expectations"><a class="header" href="#controlling-passfail-expectations">Controlling pass/fail expectations</a></h2>
|
|
<p>By default, a UI test is expected to <strong>generate a compile error</strong> because most
|
|
of the tests are checking for invalid input and error diagnostics. However, you
|
|
can also make UI tests where compilation is expected to succeed, and you can
|
|
even run the resulting program. Just add one of the following
|
|
<a href="directives.html">directives</a>:</p>
|
|
<ul>
|
|
<li>Pass directives:
|
|
<ul>
|
|
<li><code>//@ check-pass</code> — compilation should succeed but skip codegen
|
|
(which is expensive and isn't supposed to fail in most cases).</li>
|
|
<li><code>//@ build-pass</code> — compilation and linking should succeed but do
|
|
not run the resulting binary.</li>
|
|
<li><code>//@ run-pass</code> — compilation should succeed and running the resulting
|
|
binary should also succeed.</li>
|
|
</ul>
|
|
</li>
|
|
<li>Fail directives:
|
|
<ul>
|
|
<li><code>//@ check-fail</code> — compilation should fail (the codegen phase is skipped).
|
|
This is the default for UI tests.</li>
|
|
<li><code>//@ build-fail</code> — compilation should fail during the codegen phase.
|
|
This will run <code>rustc</code> twice, once to verify that it compiles successfully
|
|
without the codegen phase, then a second time the full compile should
|
|
fail.</li>
|
|
<li><code>//@ run-fail</code> — compilation should succeed, but running the resulting
|
|
binary should fail.</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<p>For <code>run-pass</code> and <code>run-fail</code> tests, by default the output of the program itself
|
|
is not checked.</p>
|
|
<p>If you want to check the output of running the program, include the
|
|
<code>check-run-results</code> directive. This will check for a <code>.run.stderr</code> and
|
|
<code>.run.stdout</code> files to compare against the actual output of the program.</p>
|
|
<p>Tests with the <code>*-pass</code> directives can be overridden with the <code>--pass</code>
|
|
command-line option:</p>
|
|
<pre><code class="language-sh">./x test tests/ui --pass check
|
|
</code></pre>
|
|
<p>The <code>--pass</code> option only affects UI tests. Using <code>--pass check</code> can run the UI
|
|
test suite much faster (roughly twice as fast on my system), though obviously
|
|
not exercising as much.</p>
|
|
<p>The <code>ignore-pass</code> directive can be used to ignore the <code>--pass</code> CLI flag if the
|
|
test won't work properly with that override.</p>
|
|
<h2 id="known-bugs"><a class="header" href="#known-bugs">Known bugs</a></h2>
|
|
<p>The <code>known-bug</code> directive may be used for tests that demonstrate a known bug
|
|
that has not yet been fixed. Adding tests for known bugs is helpful for several
|
|
reasons, including:</p>
|
|
<ol>
|
|
<li>Maintaining a functional test that can be conveniently reused when the bug is
|
|
fixed.</li>
|
|
<li>Providing a sentinel that will fail if the bug is incidentally fixed. This
|
|
can alert the developer so they know that the associated issue has been fixed
|
|
and can possibly be closed.</li>
|
|
</ol>
|
|
<p>This directive takes comma-separated issue numbers as arguments, or <code>"unknown"</code>:</p>
|
|
<ul>
|
|
<li><code>//@ known-bug: #123, #456</code> (when the issues are on rust-lang/rust)</li>
|
|
<li><code>//@ known-bug: rust-lang/chalk#123456</code>
|
|
(allows arbitrary text before the <code>#</code>, which is useful when the issue is on another repo)</li>
|
|
<li><code>//@ known-bug: unknown</code>
|
|
(when there is no known issue yet; preferrably open one if it does not already exist)</li>
|
|
</ul>
|
|
<p>Do not include <a href="#error-annotations">error annotations</a> in a test with
|
|
<code>known-bug</code>. The test should still include other normal directives and
|
|
stdout/stderr files.</p>
|
|
<h2 id="test-organization"><a class="header" href="#test-organization">Test organization</a></h2>
|
|
<p>When deciding where to place a test file, please try to find a subdirectory that
|
|
best matches what you are trying to exercise. Do your best to keep things
|
|
organized. Admittedly it can be difficult as some tests can overlap different
|
|
categories, and the existing layout may not fit well.</p>
|
|
<p>Name the test by a concise description of what the test is checking. Avoid
|
|
including the issue number in the test name. See <a href="best-practices.html">best
|
|
practices</a> for a more in-depth discussion of this.</p>
|
|
<p>Ideally, the test should be added to a directory that helps identify what piece
|
|
of code is being tested here (e.g.,
|
|
<code>tests/ui/borrowck/reject-move-out-of-borrow-via-pat.rs</code>)</p>
|
|
<p>When writing a new feature, you may want to <strong>create a subdirectory to store
|
|
your tests</strong>. For example, if you are implementing RFC 1234 ("Widgets"), then it
|
|
might make sense to put the tests in a directory like
|
|
<code>tests/ui/rfc1234-widgets/</code>.</p>
|
|
<p>In other cases, there may already be a suitable directory.</p>
|
|
<p>Over time, the <a href="https://github.com/rust-lang/rust/blob/master/tests/ui"><code>tests/ui</code></a> directory has grown very fast. There is a check in
|
|
<a href="intro.html#tidy">tidy</a> that will ensure none of the subdirectories has more than
|
|
1000 entries. Having too many files causes problems because it isn't editor/IDE
|
|
friendly and the GitHub UI won't show more than 1000 entries. However, since
|
|
<code>tests/ui</code> (UI test root directory) and <code>tests/ui/issues</code> directories have more
|
|
than 1000 entries, we set a different limit for those directories. So, please
|
|
avoid putting a new test there and try to find a more relevant place.</p>
|
|
<p>For example, if your test is related to closures, you should put it in
|
|
<code>tests/ui/closures</code>. When you reach the limit, you could increase it by tweaking
|
|
<a href="https://github.com/rust-lang/rust/blob/master/src/tools/tidy/src/ui_tests.rs">here</a>.</p>
|
|
<h2 id="rustfix-tests"><a class="header" href="#rustfix-tests">Rustfix tests</a></h2>
|
|
<p>UI tests can validate that diagnostic suggestions apply correctly and that the
|
|
resulting changes compile correctly. This can be done with the <code>run-rustfix</code>
|
|
directive:</p>
|
|
<pre><code class="language-rust ignore">//@ run-rustfix
|
|
//@ check-pass
|
|
#![crate_type = "lib"]
|
|
|
|
pub struct not_camel_case {}
|
|
//~^ WARN `not_camel_case` should have an upper camel case name
|
|
//~| HELP convert the identifier to upper camel case
|
|
//~| SUGGESTION NotCamelCase</code></pre>
|
|
<p>Rustfix tests should have a file with the <code>.fixed</code> extension which contains the
|
|
source file after the suggestion has been applied.</p>
|
|
<ul>
|
|
<li>When the test is run, compiletest first checks that the correct lint/warning
|
|
is generated.</li>
|
|
<li>Then, it applies the suggestion and compares against <code>.fixed</code> (they must
|
|
match).</li>
|
|
<li>Finally, the fixed source is compiled, and this compilation is required to
|
|
succeed.</li>
|
|
</ul>
|
|
<p>Usually when creating a rustfix test you will generate the <code>.fixed</code> file
|
|
automatically with the <code>x test --bless</code> option.</p>
|
|
<p>The <code>run-rustfix</code> directive will cause <em>all</em> suggestions to be applied, even if
|
|
they are not <a href="../diagnostics.html#suggestions"><code>MachineApplicable</code></a>. If this is a
|
|
problem, then you can add the <code>rustfix-only-machine-applicable</code> directive in
|
|
addition to <code>run-rustfix</code>. This should be used if there is a mixture of
|
|
different suggestion levels, and some of the non-machine-applicable ones do not
|
|
apply cleanly.</p>
|
|
<h2 id="compare-modes"><a class="header" href="#compare-modes">Compare modes</a></h2>
|
|
<p><a href="compiletest.html#compare-modes">Compare modes</a> can be used to run all tests with
|
|
different flags from what they are normally compiled with. In some cases, this
|
|
might result in different output from the compiler. To support this, different
|
|
output files can be saved which contain the output based on the compare mode.</p>
|
|
<p>For example, when using the Polonius mode, a test <code>foo.rs</code> will first look for
|
|
expected output in <code>foo.polonius.stderr</code>, falling back to the usual <code>foo.stderr</code>
|
|
if not found. This is useful as different modes can sometimes result in
|
|
different diagnostics and behavior. This can help track which tests have
|
|
differences between the modes, and to visually inspect those diagnostic
|
|
differences.</p>
|
|
<p>If in the rare case you encounter a test that has different behavior, you can
|
|
run something like the following to generate the alternate stderr file:</p>
|
|
<pre><code class="language-sh">./x test tests/ui --compare-mode=polonius --bless
|
|
</code></pre>
|
|
<p>Currently none of the compare modes are checked in CI for UI tests.</p>
|
|
<h2 id="rustc_-test-attributes"><a class="header" href="#rustc_-test-attributes"><code>rustc_*</code> TEST attributes</a></h2>
|
|
<p>The compiler defines several perma-unstable <code>#[rustc_*]</code> attributes gated behind
|
|
the internal feature <code>rustc_attrs</code> that dump extra compiler-internal
|
|
information. See the corresponding subsection in <a href="../compiler-debugging.html#rustc_-test-attributes">compiler debugging</a> for more
|
|
details.</p>
|
|
<p>They can be used in tests to more precisely, legibly and easily test internal
|
|
compiler state in cases where it would otherwise be very hard to do the same
|
|
with "user-facing" Rust alone. Indeed, one could say that this slightly abuses
|
|
the term "UI" (<em>user</em> interface) and turns such UI tests from black-box tests
|
|
into white-box ones. Use them carefully and sparingly.</p>
|
|
<h2 id="ui-test-mode-preset-lint-levels"><a class="header" href="#ui-test-mode-preset-lint-levels">UI test mode preset lint levels</a></h2>
|
|
<p>By default, test suites under UI test mode (<code>tests/ui</code>, <code>tests/ui-fulldeps</code>,
|
|
but not <code>tests/rustdoc-ui</code>) will specify</p>
|
|
<ul>
|
|
<li><code>-A unused</code></li>
|
|
<li><code>-A internal_features</code></li>
|
|
</ul>
|
|
<p>If:</p>
|
|
<ul>
|
|
<li>The ui test's pass mode is below <code>run</code> (i.e. check or build).</li>
|
|
<li>No compare modes are specified.</li>
|
|
</ul>
|
|
<p>Since they can be very noisy in ui tests.</p>
|
|
<p>You can override them with <code>compile-flags</code> lint level flags or
|
|
in-source lint level attributes as required.</p>
|
|
<p>Note that the <code>rustfix</code> version will <em>not</em> have <code>-A unused</code> passed,
|
|
meaning that you may have to <code>#[allow(unused)]</code> to suppress <code>unused</code>
|
|
lints on the rustfix'd file (because we might be testing rustfix
|
|
on <code>unused</code> lints themselves).</p>
|
|
<hr>
|
|
<ol class="footnote-definition"><li id="footnote-main test file">
|
|
<p>This is a file that has the <code>~?</code> annotations,
|
|
as distinct from aux files, or sources that we have no control over. <a href="#fr-main test file-1">↩</a></p>
|
|
</li>
|
|
</ol>
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../tests/compiletest.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../tests/directives.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../tests/compiletest.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../tests/directives.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js"></script>
|
|
<script src="../mark.min.js"></script>
|
|
<script src="../searcher.js"></script>
|
|
|
|
<script src="../clipboard.min.js"></script>
|
|
<script src="../highlight.js"></script>
|
|
<script src="../book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
<script src="../mermaid.min.js"></script>
|
|
<script src="../mermaid-init.js"></script>
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|