391 lines
20 KiB
HTML
391 lines
20 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="light sidebar-visible" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>Adding new tests - Rust Compiler Development Guide</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="A guide to developing the Rust compiler (rustc)">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
|
|
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
|
|
|
|
<!-- Provide site root and default themes to javascript -->
|
|
<script>
|
|
const path_to_root = "../";
|
|
const default_light_theme = "light";
|
|
const default_dark_theme = "navy";
|
|
</script>
|
|
<!-- Start loading toc.js asap -->
|
|
<script src="../toc.js"></script>
|
|
</head>
|
|
<body>
|
|
<div id="body-container">
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
let theme = localStorage.getItem('mdbook-theme');
|
|
let sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
|
let theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
const html = document.documentElement;
|
|
html.classList.remove('light')
|
|
html.classList.add(theme);
|
|
html.classList.add("js");
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
let sidebar = null;
|
|
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<!-- populated by js -->
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
|
<noscript>
|
|
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
|
|
</noscript>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
|
<div class="sidebar-resize-indicator"></div>
|
|
</div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Rust Compiler Development Guide</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
<a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
|
</a>
|
|
<a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/tests/adding.md" title="Suggest an edit" aria-label="Suggest an edit">
|
|
<i id="git-edit-button" class="fa fa-edit"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<h1 id="adding-new-tests"><a class="header" href="#adding-new-tests">Adding new tests</a></h1>
|
|
<ul>
|
|
<li><a href="#ui-test-walkthrough">UI test walkthrough</a>
|
|
<ul>
|
|
<li><a href="#step-1-add-a-test-file">Step 1: Add a test file</a></li>
|
|
<li><a href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></li>
|
|
<li><a href="#step-3-add-error-annotations">Step 3: Add error annotations</a></li>
|
|
<li><a href="#step-4-review-the-output">Step 4: Review the output</a></li>
|
|
<li><a href="#step-5-check-other-tests">Step 5: Check other tests</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></li>
|
|
</ul>
|
|
<p><strong>In general, we expect every PR that fixes a bug in rustc to come accompanied
|
|
by a regression test of some kind.</strong> This test should fail in master but pass
|
|
after the PR. These tests are really useful for preventing us from repeating the
|
|
mistakes of the past.</p>
|
|
<p>The first thing to decide is which kind of test to add. This will depend on the
|
|
nature of the change and what you want to exercise. Here are some rough
|
|
guidelines:</p>
|
|
<ul>
|
|
<li>The majority of compiler tests are done with <a href="compiletest.html">compiletest</a>.
|
|
<ul>
|
|
<li>The majority of compiletest tests are <a href="ui.html">UI</a> tests in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
|
|
directory.</li>
|
|
</ul>
|
|
</li>
|
|
<li>Changes to the standard library are usually tested within the standard library
|
|
itself.
|
|
<ul>
|
|
<li>The majority of standard library tests are written as doctests, which
|
|
illustrate and exercise typical API behavior.</li>
|
|
<li>Additional <a href="intro.html#package-tests">unit tests</a> should go in
|
|
<code>library/${crate}/tests</code> (where <code>${crate}</code> is usually <code>core</code>, <code>alloc</code>, or
|
|
<code>std</code>).</li>
|
|
</ul>
|
|
</li>
|
|
<li>If the code is part of an isolated system, and you are not testing compiler
|
|
output, consider using a <a href="intro.html#package-tests">unit or integration test</a>.</li>
|
|
<li>Need to run rustdoc? Prefer a <code>rustdoc</code> or <code>rustdoc-ui</code> test. Occasionally
|
|
you'll need <code>rustdoc-js</code> as well.</li>
|
|
<li>Other compiletest test suites are generally used for special purposes:
|
|
<ul>
|
|
<li>Need to run gdb or lldb? Use the <code>debuginfo</code> test suite.</li>
|
|
<li>Need to inspect LLVM IR or MIR IR? Use the <code>codegen</code> or <code>mir-opt</code> test
|
|
suites.</li>
|
|
<li>Need to inspect the resulting binary in some way? Or if all the other test
|
|
suites are too limited for your purposes? Then use <code>run-make</code>.</li>
|
|
<li>Check out the <a href="compiletest.html">compiletest</a> chapter for more specialized test suites.</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<p>After deciding on which kind of test to add, see <a href="best-practices.html">best
|
|
practices</a> for guidance on how to author tests that are easy
|
|
to work with that stand the test of time (i.e. if a test fails or need to be
|
|
modified several years later, how can we make it easier for them?).</p>
|
|
<h2 id="ui-test-walkthrough"><a class="header" href="#ui-test-walkthrough">UI test walkthrough</a></h2>
|
|
<p>The following is a basic guide for creating a <a href="ui.html">UI test</a>, which is one of
|
|
the most common compiler tests. For this tutorial, we'll be adding a test for an
|
|
async error message.</p>
|
|
<h3 id="step-1-add-a-test-file"><a class="header" href="#step-1-add-a-test-file">Step 1: Add a test file</a></h3>
|
|
<p>The first step is to create a Rust source file somewhere in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
|
|
tree. When creating a test, do your best to find a good location and name (see
|
|
<a href="ui.html#test-organization">Test organization</a> for more). Since naming is the
|
|
hardest part of development, everything should be downhill from here!</p>
|
|
<p>Let's place our async test at <code>tests/ui/async-await/await-without-async.rs</code>:</p>
|
|
<pre><code class="language-rust ignore">// Provide diagnostics when the user writes `await` in a non-`async` function.
|
|
//@ edition:2018
|
|
|
|
async fn foo() {}
|
|
|
|
fn bar() {
|
|
foo().await
|
|
}
|
|
|
|
fn main() {}</code></pre>
|
|
<p>A few things to notice about our test:</p>
|
|
<ul>
|
|
<li>The top should start with a short comment that <a href="#explanatory_comment">explains what the test is
|
|
for</a>.</li>
|
|
<li>The <code>//@ edition:2018</code> comment is called a <a href="directives.html">directive</a> which
|
|
provides instructions to compiletest on how to build the test. Here we need to
|
|
set the edition for <code>async</code> to work (the default is edition 2015).</li>
|
|
<li>Following that is the source of the test. Try to keep it succinct and to the
|
|
point. This may require some effort if you are trying to minimize an example
|
|
from a bug report.</li>
|
|
<li>We end this test with an empty <code>fn main</code> function. This is because the default
|
|
for UI tests is a <code>bin</code> crate-type, and we don't want the "main not found"
|
|
error in our test. Alternatively, you could add <code>#![crate_type="lib"]</code>.</li>
|
|
</ul>
|
|
<h3 id="step-2-generate-the-expected-output"><a class="header" href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></h3>
|
|
<p>The next step is to create the expected output snapshots from the compiler. This
|
|
can be done with the <code>--bless</code> option:</p>
|
|
<pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs --bless
|
|
</code></pre>
|
|
<p>This will build the compiler (if it hasn't already been built), compile the
|
|
test, and place the output of the compiler in a file called
|
|
<code>tests/ui/async-await/await-without-async.stderr</code>.</p>
|
|
<p>However, this step will fail! You should see an error message, something like
|
|
this:</p>
|
|
<blockquote>
|
|
<p>error: /rust/tests/ui/async-await/await-without-async.rs:7: unexpected
|
|
error: '7:10: 7:16: <code>await</code> is only allowed inside <code>async</code> functions and
|
|
blocks E0728'</p>
|
|
</blockquote>
|
|
<p>This is because the stderr contains errors which were not matched by error
|
|
annotations in the source file.</p>
|
|
<h3 id="step-3-add-error-annotations"><a class="header" href="#step-3-add-error-annotations">Step 3: Add error annotations</a></h3>
|
|
<p>Every error needs to be annotated with a comment in the source with the text of
|
|
the error. In this case, we can add the following comment to our test file:</p>
|
|
<pre><code class="language-rust ignore">fn bar() {
|
|
foo().await
|
|
//~^ ERROR `await` is only allowed inside `async` functions and blocks
|
|
}</code></pre>
|
|
<p>The <code>//~^</code> squiggle caret comment tells compiletest that the error belongs to
|
|
the <em>previous</em> line (more on this in the <a href="ui.html#error-annotations">Error
|
|
annotations</a> section).</p>
|
|
<p>Save that, and run the test again:</p>
|
|
<pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs
|
|
</code></pre>
|
|
<p>It should now pass, yay!</p>
|
|
<h3 id="step-4-review-the-output"><a class="header" href="#step-4-review-the-output">Step 4: Review the output</a></h3>
|
|
<p>Somewhat hand-in-hand with the previous step, you should inspect the <code>.stderr</code>
|
|
file that was created to see if it looks like how you expect. If you are adding
|
|
a new diagnostic message, now would be a good time to also consider how readable
|
|
the message looks overall, particularly for people new to Rust.</p>
|
|
<p>Our example <code>tests/ui/async-await/await-without-async.stderr</code> file should look
|
|
like this:</p>
|
|
<pre><code class="language-text">error[E0728]: `await` is only allowed inside `async` functions and blocks
|
|
--> $DIR/await-without-async.rs:7:10
|
|
|
|
|
LL | fn bar() {
|
|
| --- this is not `async`
|
|
LL | foo().await
|
|
| ^^^^^^ only allowed inside `async` functions and blocks
|
|
|
|
error: aborting due to previous error
|
|
|
|
For more information about this error, try `rustc --explain E0728`.
|
|
</code></pre>
|
|
<p>You may notice some things look a little different than the regular compiler
|
|
output.</p>
|
|
<ul>
|
|
<li>The <code>$DIR</code> removes the path information which will differ between systems.</li>
|
|
<li>The <code>LL</code> values replace the line numbers. That helps avoid small changes in
|
|
the source from triggering large diffs. See the
|
|
<a href="ui.html#normalization">Normalization</a> section for more.</li>
|
|
</ul>
|
|
<p>Around this stage, you may need to iterate over the last few steps a few times
|
|
to tweak your test, re-bless the test, and re-review the output.</p>
|
|
<h3 id="step-5-check-other-tests"><a class="header" href="#step-5-check-other-tests">Step 5: Check other tests</a></h3>
|
|
<p>Sometimes when adding or changing a diagnostic message, this will affect other
|
|
tests in the test suite. The final step before posting a PR is to check if you
|
|
have affected anything else. Running the UI suite is usually a good start:</p>
|
|
<pre><code class="language-sh">./x test tests/ui
|
|
</code></pre>
|
|
<p>If other tests start failing, you may need to investigate what has changed and
|
|
if the new output makes sense.</p>
|
|
<p>You may also need to re-bless the output with the <code>--bless</code> flag.</p>
|
|
<p><a id="explanatory_comment"></a></p>
|
|
<h2 id="comment-explaining-what-the-test-is-about"><a class="header" href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></h2>
|
|
<p>The first comment of a test file should <strong>summarize the point of the test</strong>, and
|
|
highlight what is important about it. If there is an issue number associated
|
|
with the test, include the issue number.</p>
|
|
<p>This comment doesn't have to be super extensive. Just something like "Regression
|
|
test for #18060: match arms were matching in the wrong order." might already be
|
|
enough.</p>
|
|
<p>These comments are very useful to others later on when your test breaks, since
|
|
they often can highlight what the problem is. They are also useful if for some
|
|
reason the tests need to be refactored, since they let others know which parts
|
|
of the test were important. Often a test must be rewritten because it no longer
|
|
tests what it was meant to test, and then it's useful to know what it <em>was</em>
|
|
meant to test exactly.</p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../tests/ci.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../tests/best-practices.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../tests/ci.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../tests/best-practices.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js"></script>
|
|
<script src="../mark.min.js"></script>
|
|
<script src="../searcher.js"></script>
|
|
|
|
<script src="../clipboard.min.js"></script>
|
|
<script src="../highlight.js"></script>
|
|
<script src="../book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
<script src="../mermaid.min.js"></script>
|
|
<script src="../mermaid-init.js"></script>
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|