Property-based testing with Hypothesis

David R. MacIver / drmaciver.com

hypothesis.works


from dateutil.parser import parse

from hypothesis import given, settings
from hypothesis.extra.datetime import datetimes


@given(datetimes())
def test_can_parse_iso_format(dt):
    formatted = dt.isoformat()
    assert formatted == parse(formatted).isoformat()



E       assert '0005-01-01T00:00:05' == '0001-05-01T00:00:05'
E         - 0005-01-01T00:00:05
E         ?    ^  ^
E         + 0001-05-01T00:00:05
E         ?    ^  ^

Falsifying example: test_can_parse_iso_format(
  dt=datetime.datetime(5, 1, 1, 0, 0, 5)
)

fluffy content begins now

(the important bits)

content warning: cynicism

writing software is easy!


print("Hello World")

But is it correct?

What even is correct software?

  • Matches the spec.
  • No bugs.
  • Fictional.

writing correct software is...

2006: impossible!

2007: easy!

2010: hard...

2015: impossible!

but verification?!?

guilt-free development

The difficult... We'll do right now.

The impossible... Seems like way too much work TBH.

bugs don't matter

useful software matters

...and bugs aren't useful

correct software

useful software

mostly working software

warning: made up graphs

testing is boring and difficult
...but it doesn't have to be.
</fluffy>

Password Requirements

  • Must be at least eight characters.
  • Must contain a lower-case letter, an upper-case letter, a number and a symbol.
  • All passwords satisfying these are valid.

Thanks to JR Heard for this example

http://blog.jrheard.com/hypothesis-and-pexpect


def is_good_password(s):
  return True # Eh, probably good enough


def test_short_is_not_allowed():
    assert not is_good_password("a")


def test_requires_character_of_each_type():
    assert not is_good_password("aaaaaaa")

    def test_short_is_not_allowed():
>       assert not is_good_password("a")
E       AssertionError: assert not True
E        +  where True = is_good_password('a')

    def test_requires_character_of_each_type():
>       assert not is_good_password("aaaaaaa")
E       AssertionError: assert not True
E        +  where True = is_good_password('aaaaaaa')


def is_good_password(s):
  return False # Passwords are a bad security model anyway


def test_allows_password_satisfying_the_rules():
    assert is_good_password("aaaaaA1!")

    def test_allows_password_satisfying_the_rules():
>       assert is_good_password("aaaaaA1!")
E       AssertionError: assert False
E        +  where False = is_good_password('aaaaaA1!')


lower_letters = "abcdefghijklmnopqrstuvwyz"
upper_letters = lower_letters.upper()
letters = lower_letters + upper_letters
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    chars = set(pw)
    return not (
        chars.isdisjoint(letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )

have we written enough tests?

of course not

oh no, we have to write more?

let's try something different


from hypothesis import given, strategies as st

SYMBOLS = '!@#$%^&*()-_=+.,'
VALID_PASSWORD_CHARS = string.ascii_letters + string.digits + SYMBOLS

@given(st.text(alphabet=VALID_PASSWORD_CHARS, max_size=7))
def test_short_passwords_are_all_invalid(s):
    assert not is_good_password(s)

    @given(st.text(alphabet=VALID_PASSWORD_CHARS, max_size=7))
    def test_short_passwords_are_all_invalid(s):
>       assert not is_good_password(s)
E       AssertionError: assert not True
E        +  where True = is_good_password('a0!')

Falsifying example: test_short_passwords_are_all_invalid(s='a0!')


lower_letters = "abcdefghijklmnopqrstuvwyz"
upper_letters = lower_letters.upper()
letters = lower_letters + upper_letters
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    if len(pw) < 8:
        return False
    chars = set(pw)
    return not (
        chars.isdisjoint(letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )

so it works now, right?


def is_good_password(pw):
    return pw == "aaaaaA1!" # Objectively the best password


REQUIRED = [string.ascii_lowercase, string.ascii_uppercase,
            string.digits, SYMBOLS]

@st.composite
def valid_password(draw):
    bits = ''.join(
      draw(st.text(alphabet=required, min_size=1))
      for required in REQUIRED)
    bits += draw(st.text(
        alphabet=VALID_PASSWORD_CHARS,
        min_size=max(0, 8 - len(bits))))
    return ''.join(draw(st.permutations(bits)))


@given(valid_password())
def test_all_valid_passwords_are_good(s):
    assert is_good_password(s)

    @given(valid_password())
    def test_all_valid_passwords_are_good(s):
>       assert is_good_password(s)
E       AssertionError: assert False
E        +  where False = is_good_password('x0!!!!!0')

Falsifying example: test_all_valid_passwords_are_good(s='x0!!!!!0')

NB: cheating


lower_letters = "abcdefghijklmnopqrstuvwyz"
upper_letters = lower_letters.upper()
letters = lower_letters + upper_letters
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    if len(pw) < 8:
        return False
    chars = set(pw)
    return not (
        chars.isdisjoint(letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )


lower_letters = "abcdefghijklmnopqrstuvwxyz"
upper_letters = lower_letters.upper()
letters = lower_letters + upper_letters
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    if len(pw) < 8:
        return False
    chars = set(pw)
    return not (
        chars.isdisjoint(letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )

one last thing


@given(st.data())
def test_all_required_parts_are_needed(data):
    parts = list(REQUIRED)
    parts.remove(
        data.draw(st.sampled_from(parts), label="removed"))
    password = data.draw(st.text(
        alphabet=''.join(parts), min_size=8), label="password")
    assert not is_good_password(password)

>       assert not is_good_password(password)
E       AssertionError: assert not True
E        +  where True = is_good_password('AAAAAA0!')

Falsifying example: test_all_required_parts_are_needed(data=data(...))
Draw 1 (removed): 'abcdefghijklmnopqrstuvwxyz'
Draw 2 (password): 'AAAAAA0!'


lower_letters = "abcdefghijklmnopqrstuvwxyz"
upper_letters = lower_letters.upper()
letters = lower_letters + upper_letters
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    if len(pw) < 8:
        return False
    chars = set(pw)
    return not (
        chars.isdisjoint(letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )


lower_letters = "abcdefghijklmnopqrstuvwxyz"
upper_letters = lower_letters.upper()
digits = "0123456789"
symbols = '!@#$%^&*()-_=+.,'


def is_good_password(pw):
    if len(pw) < 8:
        return False
    chars = set(pw)
    return not (
        chars.isdisjoint(lower_letters) or
        chars.isdisjoint(upper_letters) or
        chars.isdisjoint(digits) or
        chars.isdisjoint(symbols)
    )

have we written enough tests now?

eh, 's probably good enough

What did we do?

  1. Took a specification.
  2. Turned each requirement in it into tests.
  3. Including both positive and negative requirements!
  4. Generated examples as input to those tests.

this finds real bugs!

...but probably not all of them.

David R. MacIver / drmaciver.com

hypothesis.works