{
  "schema": "shlex-best-fit-publication-v1",
  "slug": "shlex-best-fit-search-2026-09-20",
  "title": "Shlex 0.9.0: nothing you type is required, and a small model that offers other words",
  "date": "2026-09-20",
  "conclusion": "On a board nobody who built the fix wrote, the holdout half goes from 19 of 117 cases passed with Shlex 0.8.4 to 62 of 117 with 0.9.0, by executed and ranked tests; every older board holds.",
  "scoring": "executed_broker_outcomes_ranked",
  "token_loss_used": false,
  "board": {
    "kept": 237,
    "dropped": {
      "review_incomplete": 1,
      "wanted_disagree": 54
    },
    "design": {
      "cases": 120,
      "sha256": "5164618e2941582b698fd1f115e81fb45370a1b1679b78320661084ea87661b1"
    },
    "holdout": {
      "cases": 117,
      "sha256": "d3c11d2f5520d8f6d52e4f9521951386b382bbf178e1f5b30a5e511accdd7076"
    },
    "families": {
      "decoys": 22,
      "glued": 18,
      "holds_all": 25,
      "meaning": 45,
      "number": 22,
      "spread": 20,
      "standard_folders": 18,
      "two_kinds": 24,
      "typo": 19,
      "year": 24
    }
  },
  "design": {
    "before": {
      "families": {
        "decoys": [
          3,
          11
        ],
        "glued": [
          0,
          9
        ],
        "holds_all": [
          6,
          13
        ],
        "meaning": [
          0,
          23
        ],
        "number": [
          1,
          11
        ],
        "spread": [
          0,
          10
        ],
        "standard_folders": [
          0,
          9
        ],
        "two_kinds": [
          1,
          12
        ],
        "typo": [
          0,
          10
        ],
        "year": [
          0,
          12
        ]
      },
      "never_shown": 7,
      "nothing_shown": 71,
      "passed": 11,
      "refused": 2,
      "total": 120
    },
    "app": {
      "families": {
        "decoys": [
          3,
          11
        ],
        "glued": [
          3,
          9
        ],
        "holds_all": [
          12,
          13
        ],
        "meaning": [
          4,
          23
        ],
        "number": [
          8,
          11
        ],
        "spread": [
          10,
          10
        ],
        "standard_folders": [
          5,
          9
        ],
        "two_kinds": [
          7,
          12
        ],
        "typo": [
          4,
          10
        ],
        "year": [
          5,
          12
        ]
      },
      "never_shown": 24,
      "nothing_shown": 8,
      "passed": 61,
      "refused": 2,
      "total": 120
    }
  },
  "holdout": {
    "before": {
      "families": {
        "decoys": [
          2,
          11
        ],
        "glued": [
          1,
          9
        ],
        "holds_all": [
          9,
          12
        ],
        "meaning": [
          0,
          22
        ],
        "number": [
          1,
          11
        ],
        "spread": [
          0,
          10
        ],
        "standard_folders": [
          4,
          9
        ],
        "two_kinds": [
          1,
          12
        ],
        "typo": [
          0,
          9
        ],
        "year": [
          1,
          12
        ]
      },
      "never_shown": 3,
      "nothing_shown": 73,
      "passed": 19,
      "refused": 0,
      "total": 117
    },
    "app": {
      "families": {
        "decoys": [
          6,
          11
        ],
        "glued": [
          7,
          9
        ],
        "holds_all": [
          11,
          12
        ],
        "meaning": [
          1,
          22
        ],
        "number": [
          5,
          11
        ],
        "spread": [
          9,
          10
        ],
        "standard_folders": [
          7,
          9
        ],
        "two_kinds": [
          6,
          12
        ],
        "typo": [
          3,
          9
        ],
        "year": [
          7,
          12
        ]
      },
      "never_shown": 22,
      "nothing_shown": 7,
      "passed": 62,
      "refused": 0,
      "total": 117
    },
    "scored_once": true
  },
  "binaries_sha256": {
    "app": "43dd379cfef012bb1a7fc16df31d2685531d4bd798c09e62fe1b949d801b5e7b",
    "before": "42e0b8309b99b6d3b96f1f7660b465cbac6636f8667d473c4e8467e9ca6ffbfa"
  },
  "weights_sha256": {
    "app": "1cca31f623f03d933fd0cd640b8952c6843c4af245ebe3b72a880c7d915aa7e5",
    "before": "f5936bb37357c8c1826a45cd10f2e90a4cb355602575b03a4271fb2ee56f1f55"
  },
  "older_boards_gold_first": {
    "jeremiah_v4": {
      "refusals-design": [
        62,
        64
      ],
      "refusals-holdout": [
        46,
        48
      ],
      "format-words-board": [
        52,
        56
      ],
      "format-words-holdout": [
        35,
        38
      ],
      "format-words-overlay": [
        7,
        7
      ],
      "historical40": [
        38,
        40
      ],
      "everyday54": [
        53,
        54
      ],
      "frozen canaries": [
        2,
        2
      ],
      "frozen raw canaries": [
        2,
        2
      ],
      "frozen development": [
        56,
        56
      ],
      "frozen raw development": [
        49,
        56
      ],
      "frozen development_v3": [
        102,
        109
      ],
      "frozen raw development_v3": [
        101,
        109
      ],
      "frozen unseen300": [
        309,
        312
      ],
      "frozen raw unseen300": [
        284,
        312
      ],
      "malformed": 108
    },
    "jeremiah_v5": {
      "refusals-design": [
        62,
        64
      ],
      "refusals-holdout": [
        46,
        48
      ],
      "format-words-board": [
        52,
        56
      ],
      "format-words-holdout": [
        36,
        38
      ],
      "format-words-overlay": [
        6,
        7
      ],
      "historical40": [
        38,
        40
      ],
      "everyday54": [
        53,
        54
      ],
      "frozen canaries": [
        2,
        2
      ],
      "frozen raw canaries": [
        2,
        2
      ],
      "frozen development": [
        55,
        56
      ],
      "frozen raw development": [
        46,
        56
      ],
      "frozen development_v3": [
        106,
        109
      ],
      "frozen raw development_v3": [
        105,
        109
      ],
      "frozen unseen300": [
        307,
        312
      ],
      "frozen raw unseen300": [
        283,
        312
      ],
      "malformed": 0
    }
  },
  "meaning_bakeoff_design": {
    "none": {
      "strict": 0,
      "cases": 23,
      "wanted_shown": 0,
      "wanted": 49,
      "top3": 0,
      "nothing": 15,
      "board": 58,
      "board_cases": 120
    },
    "vectors-0.7": {
      "strict": 1,
      "cases": 23,
      "wanted_shown": 2,
      "wanted": 49,
      "top3": 2,
      "nothing": 13,
      "board": 62,
      "board_cases": 120
    },
    "grok": {
      "strict": 4,
      "cases": 23,
      "wanted_shown": 24,
      "wanted": 49,
      "top3": 12,
      "nothing": 3,
      "board": 59,
      "board_cases": 120
    },
    "documents-w0.0-k0.8": {
      "strict": 8,
      "cases": 23,
      "wanted_shown": 28,
      "wanted": 49,
      "top3": 18,
      "nothing": 0,
      "board": 53,
      "board_cases": 120
    },
    "model-v4-final": {
      "strict": 0,
      "cases": 23,
      "wanted_shown": 0,
      "wanted": 49,
      "top3": 0,
      "nothing": 15,
      "board": 55,
      "board_cases": 120
    },
    "model-v5b-1593-final": {
      "strict": 4,
      "cases": 23,
      "wanted_shown": 17,
      "wanted": 49,
      "top3": 10,
      "nothing": 6,
      "board": 61,
      "board_cases": 120
    }
  },
  "training_rows": {
    "written": 3900,
    "accepted": 3673,
    "empty_answers": 489,
    "in_training": 3296,
    "unchanged_first_question_rows": 13696,
    "dropped": {
      "board_request": 3,
      "duplicate": 223,
      "unreviewed": 1
    }
  },
  "runs": [
    {
      "name": "first pass",
      "gpu": "NVIDIA RTX 6000 Ada Generation",
      "hourly_usd": 0.84,
      "usd": 0.4,
      "checkpoints": [
        {
          "path": "meanings_v5/checkpoint-1062/model.safetensors",
          "sha256": "f5f30a6a7397feb77d8a01e604544419867aa03d65f077fdd533fef4468c395d",
          "bytes": 3087467144
        },
        {
          "path": "meanings_v5/checkpoint-265/model.safetensors",
          "sha256": "94ee8eddd3af45acd93eda2d0c8f85cfa23069d661e71207e8142557b8ce4adf",
          "bytes": 3087467144
        },
        {
          "path": "meanings_v5/checkpoint-531/model.safetensors",
          "sha256": "d54c38c0ddd3163c407619d4d2bfeeb61a701676c1045a8586cebb68400d3239",
          "bytes": 3087467144
        }
      ]
    },
    {
      "name": "second pass",
      "gpu": "NVIDIA RTX 6000 Ada Generation",
      "hourly_usd": 0.84,
      "usd": 0.72,
      "checkpoints": [
        {
          "path": "meanings_v5b/checkpoint-1062/model.safetensors",
          "sha256": "2c156d0a4c70207a9d7bd8c6497ad5b901b6ef4284ebdad88fddff9f65f64bd6",
          "bytes": 3087467144
        },
        {
          "path": "meanings_v5b/checkpoint-1593/model.safetensors",
          "sha256": "1cca31f623f03d933fd0cd640b8952c6843c4af245ebe3b72a880c7d915aa7e5",
          "bytes": 3087467144
        },
        {
          "path": "meanings_v5b/checkpoint-2124/model.safetensors",
          "sha256": "7b5d621be96bf6e98b35b057cd54c96e54c2939066869c112834a0995bccac92",
          "bytes": 3087467144
        }
      ]
    }
  ],
  "promoted_checkpoint": {
    "path": "meanings_v5b/checkpoint-1593/model.safetensors",
    "sha256": "1cca31f623f03d933fd0cd640b8952c6843c4af245ebe3b72a880c7d915aa7e5",
    "bytes": 3087467144
  },
  "gpu_deleted": true,
  "training_rows_published": false,
  "weights_published": false,
  "holdout_content_published": false,
  "generated_unix": 1789931166.975643
}