How to Order xUnit Tests and Collections

xUnit is an extremely extensible unit testing framework!

If you need to control the order of your unit tests, then all you have to do is implement an ITestCaseOrderer. Once implemented, you just add a TestCaseOrdererAttribute to the top of your test class to use it. Below we use a custom OrderAttribute to order the tests.

To control the order of the test collections you can do a very similar trick by implementing an ITestCollectionOrderer. However, an ITestCollection is not neccessarily associated with a specific class, so to to use attributes to order them you need to use a little reflection. Check out the sample below for details.

Implementation

/// <summary>

/// Used by CustomOrderer

/// </summary>

public class OrderAttribute : Attribute

{

    public int I { get; }

    public OrderAttribute(int i)

    {

        I = i;

    }

}

/// <summary>

/// Custom xUnit test collection orderer that uses the OrderAttribute

/// </summary>

public class CustomTestCollectionOrderer : ITestCollectionOrderer

{

    public const string TypeName = "xUnitCustom.CustomTestCollectionOrderer";

    public const string AssembyName = "xUnitCustom";

    public IEnumerable<ITestCollection> OrderTestCollections(

        IEnumerable<ITestCollection> testCollections)

    {

        return testCollections.OrderBy(GetOrder);

    }

    /// <summary>

    /// Test collections are not bound to a specific class, however they

    /// are named by default with the type name as a suffix. We try to

    /// get the class name from the DisplayName and then use reflection to

    /// find the class and OrderAttribute.

    /// </summary>

    private static int GetOrder(

        ITestCollection testCollection)

    {

        var i = testCollection.DisplayName.LastIndexOf(' ');

        if (i <= -1)

            return 0;

        var className = testCollection.DisplayName.Substring(i + 1);

        var type = Type.GetType(className);

        if (type == null)

            return 0;

        var attr = type.GetCustomAttribute<OrderAttribute>();

        return attr?.I ?? 0;

    }

}

/// <summary>

/// Custom xUnit test case orderer that uses the OrderAttribute

/// </summary>

public class CustomTestCaseOrderer : ITestCaseOrderer

{

    public const string TypeName = "xUnitCustom.CustomTestCaseOrderer";

    public const string AssembyName = "xUnitCustom";

    public static readonly ConcurrentDictionary<string, ConcurrentQueue<string>>

        QueuedTests = new ConcurrentDictionary<string, ConcurrentQueue<string>>();

    public IEnumerable<TTestCase> OrderTestCases<TTestCase>(

        IEnumerable<TTestCase> testCases)

        where TTestCase : ITestCase

    {

        return testCases.OrderBy(GetOrder);

    }

    private static int GetOrder<TTestCase>(

        TTestCase testCase)

        where TTestCase : ITestCase

    {

        // Enqueue the test name.

        QueuedTests

            .GetOrAdd(

                testCase.TestMethod.TestClass.Class.Name,

                key => new ConcurrentQueue<string>())

            .Enqueue(testCase.TestMethod.Method.Name);

        // Order the test based on the attribute.

        var attr = testCase.TestMethod.Method

            .ToRuntimeMethod()

            .GetCustomAttribute<OrderAttribute>();

        return attr?.I ?? 0;

    }

}

Test Assembly Configuration

[assembly: CollectionBehavior(DisableTestParallelization = true)]

[assembly: TestCollectionOrderer(

    CustomTestCollectionOrderer.TypeName, 

    CustomTestCollectionOrderer.AssembyName)]

Tests

/// <summary>

/// These tests only succeed if you run all tests in the class.

/// </summary>

[Order(1)]

public class TestClassOne : TestClassBase

{

    [Fact]

    public void Zero()

    {

        AssertTestName("Zero");

        Assert.Equal(0, I);

        Interlocked.Increment(ref I);

    }

    [Fact, Order(1)]

    public void First()

    {

        AssertTestName("First");

        Assert.Equal(1, I);

        Interlocked.Increment(ref I);

    }

}

/// <summary>

/// These tests only succeed if you run all tests in the assembly

/// (this is because it asserts that TestClassOne has already run).

/// </summary>

[Order(2)]

public class TestClassTwo : TestClassBase

{

    [Fact, Order(2)]

    public void Second()

    {

        AssertTestName("Second");

        Assert.Equal(2, I);

        Interlocked.Increment(ref I);

    }

    [Theory, Order(34), InlineData(3), InlineData(4)]

    public void ThirdAndFourth(int i)

    {

        AssertTestName("ThirdAndFourth");

        Assert.Equal(i, I);

        Interlocked.Increment(ref I);

    }

}

/// <summary>

/// These tests only succeed if you run all tests in the class.

/// </summary>

[TestCaseOrderer(

    CustomTestCaseOrderer.TypeName, 

    CustomTestCaseOrderer.AssembyName)]

public class TestClassBase

{

    protected static int I;

    protected void AssertTestName(string testName)

    {

        var type = GetType();

        var queue = CustomTestCaseOrderer.QueuedTests[type.FullName];

        string dequeuedName;

        var result = queue.TryDequeue(out dequeuedName);

        Assert.True(result);

        Assert.Equal(testName, dequeuedName);

    }

}

Enjoy,
Tom