sweedworks

← all sources

task.py

The copy task a fixed window cannot do

50 lines. This is the file the build actually runs, copied verbatim at build time.

  1. 1"""A task that cannot be done by looking at the last few characters.
  2. 2 
  3. 3Each line pairs keys with values, then asks for one of them again:
  4. 4 
  5. 5 a3 f7 c1 f -> 7
  6. 6 
  7. 7Answering needs the model to find the earlier occurrence of the queried key and
  8. 8copy what followed it. The distance back is variable, so no fixed offset works;
  9. 9the model has to match on content. This is the smallest thing I could construct
  10. 10that a fixed-window model is structurally unable to do and attention can.
  11. 11"""
  12. 12 
  13. 13import random
  14. 14 
  15. 15KEYS = "abcdef"
  16. 16VALUES = "0123456789"
  17. 17PAIRS = 3
  18. 18 
  19. 19 
  20. 20def make_line(rng):
  21. 21 keys = rng.sample(KEYS, PAIRS)
  22. 22 values = [rng.choice(VALUES) for _ in range(PAIRS)]
  23. 23 query = rng.choice(range(PAIRS))
  24. 24 prompt = " ".join(k + v for k, v in zip(keys, values))
  25. 25 return f"{prompt} {keys[query]}", values[query]
  26. 26 
  27. 27 
  28. 28def make_corpus(n, seed=1):
  29. 29 """Returns (text, list of (prompt, answer)) — the text is the lines joined."""
  30. 30 rng = random.Random(seed)
  31. 31 items = [make_line(rng) for _ in range(n)]
  32. 32 text = "".join(f"{p}{a}\n" for p, a in items)
  33. 33 return text, items
  34. 34 
  35. 35 
  36. 36def accuracy(predict, items):
  37. 37 """predict(prompt) -> character. Fraction of answers exactly right."""
  38. 38 right = 0
  39. 39 for prompt, answer in items:
  40. 40 if predict(prompt) == answer:
  41. 41 right += 1
  42. 42 return right / len(items)
  43. 43 
  44. 44 
  45. 45if __name__ == "__main__":
  46. 46 text, items = make_corpus(6, seed=3)
  47. 47 print(text.rstrip())
  48. 48 print()
  49. 49 print("chance accuracy is 1/10 =", 0.1)