{ "schema": "shlex-jeremiah-v4-public-v1", "title": "Jeremiah v4: rows for three string rules, measured on a 312-case unseen board", "date": "2026-09-18", "conclusion": "v4 step 856 passes 284/312 (91%) of a 312-case board whose providers, topics, repos and nouns were never trained on (95% interval 87 to 94%); the model in the app passes 239/312 (77%) (72 to 81%). On the unseen v3 development cases 856 passes 88/96 (92%) against 76/96 (79%); canaries 2/2 (100%); zero false acceptances and zero unsafe executions on every board. The v2 development and regression boards are not yet scored, so no snapshot has passed the full gate set; jeremiah_v4 ships in Shlex 0.7.0 and 0.7.1 as an operator-authorized field test. The run cost 0.36 USD on a rented RTX 6000 Ada, created on its first attempt.", "update": "Update, the same day: the plural and spelling rule this note proposed now runs in the app (Shlex 0.7.1), not in the model. Replaying the same cached model lines through it, v4 step 856 passes 304/312 (97%) of the unseen board (95 to 99%) and the model of the previous round passes 289/312 (93%), more than this run's training gave it; no case newly fails for any of the four models and there are still zero false acceptances and zero unsafe executions.", "updated": "2026-09-18", "nominee": null, "best_new_snapshot": "v4-856", "field_test_release": { "app": "Shlex 0.7.1", "model": "jeremiah_v4", "snapshot": "v4-856", "previous": "Shlex 0.7.0" }, "app_rule_replay": { "what": "cached raw model lines passed through the app's topic rule and re-executed; raw lines are the ones scored in models", "rule": [ "adds the request's spelling of an edited stem", "adds singular, regular plural and irregular plural of each final word", "only adds to any; never removes; exact (quoted or capitalised) phrases untouched" ], "boards": { "v3-825-app": { "canaries": { "total": 2, "passed_raw": 2, "passed": 2, "interval": [ 0.3424, 1.0 ], "changed_lines": 0, "fixed": 0, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "operator_canary": [ 2, 2, 2 ] } }, "development": { "total": 56, "passed_raw": 42, "passed": 53, "interval": [ 0.8539, 0.9816 ], "changed_lines": 15, "fixed": 11, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "common_word_providers": [ 9, 9, 9 ], "compound_providers": [ 16, 16, 18 ], "document_nouns_limit": [ 6, 6, 6 ], "refusal_pairs": [ 6, 6, 6 ], "topic_plurals": [ 5, 16, 17 ] } }, "development_v3": { "total": 109, "passed_raw": 89, "passed": 95, "interval": [ 0.7959, 0.9219 ], "changed_lines": 6, "fixed": 6, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 18, 18, 18 ], "document_nouns": [ 14, 14, 16 ], "refusal_pairs": [ 17, 17, 17 ], "scope_synonyms": [ 10, 12, 18 ], "secret_spellings": [ 11, 11, 17 ], "topic_alternatives": [ 19, 23, 23 ] } }, "unseen300": { "total": 312, "passed_raw": 239, "passed": 289, "interval": [ 0.8918, 0.9504 ], "changed_lines": 66, "fixed": 50, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 7, 7, 9 ], "common_word_providers": [ 39, 39, 40 ], "compound_providers": [ 46, 46, 58 ], "document_nouns": [ 29, 29, 29 ], "refusal_pairs": [ 18, 18, 18 ], "scope_synonyms": [ 34, 35, 38 ], "secret_spellings": [ 17, 17, 17 ], "topic_alternatives": [ 30, 32, 33 ], "topic_plurals": [ 19, 66, 70 ] } } }, "v4-214": { "canaries": { "total": 2, "passed_raw": 2, "passed": 2, "interval": [ 0.3424, 1.0 ], "changed_lines": 0, "fixed": 0, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "operator_canary": [ 2, 2, 2 ] } }, "development": { "total": 56, "passed_raw": 48, "passed": 55, "interval": [ 0.9055, 0.9968 ], "changed_lines": 9, "fixed": 7, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "common_word_providers": [ 9, 9, 9 ], "compound_providers": [ 18, 18, 18 ], "document_nouns_limit": [ 6, 6, 6 ], "refusal_pairs": [ 6, 6, 6 ], "topic_plurals": [ 9, 16, 17 ] } }, "development_v3": { "total": 109, "passed_raw": 99, "passed": 100, "interval": [ 0.8505, 0.956 ], "changed_lines": 1, "fixed": 1, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 16, 16, 18 ], "document_nouns": [ 16, 16, 16 ], "refusal_pairs": [ 17, 17, 17 ], "scope_synonyms": [ 12, 13, 18 ], "secret_spellings": [ 16, 16, 17 ], "topic_alternatives": [ 22, 22, 23 ] } }, "unseen300": { "total": 312, "passed_raw": 272, "passed": 299, "interval": [ 0.93, 0.9755 ], "changed_lines": 36, "fixed": 27, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 8, 8, 9 ], "common_word_providers": [ 39, 39, 40 ], "compound_providers": [ 52, 52, 58 ], "document_nouns": [ 29, 29, 29 ], "refusal_pairs": [ 18, 18, 18 ], "scope_synonyms": [ 34, 35, 38 ], "secret_spellings": [ 17, 17, 17 ], "topic_alternatives": [ 31, 32, 33 ], "topic_plurals": [ 44, 69, 70 ] } } }, "v4-428": { "canaries": { "total": 2, "passed_raw": 2, "passed": 2, "interval": [ 0.3424, 1.0 ], "changed_lines": 0, "fixed": 0, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "operator_canary": [ 2, 2, 2 ] } }, "development": { "total": 56, "passed_raw": 48, "passed": 54, "interval": [ 0.8788, 0.9902 ], "changed_lines": 7, "fixed": 6, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "common_word_providers": [ 9, 9, 9 ], "compound_providers": [ 18, 18, 18 ], "document_nouns_limit": [ 6, 6, 6 ], "refusal_pairs": [ 6, 6, 6 ], "topic_plurals": [ 9, 15, 17 ] } }, "development_v3": { "total": 109, "passed_raw": 99, "passed": 102, "interval": [ 0.8733, 0.9685 ], "changed_lines": 3, "fixed": 3, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 18, 18, 18 ], "document_nouns": [ 15, 15, 16 ], "refusal_pairs": [ 17, 17, 17 ], "scope_synonyms": [ 15, 15, 18 ], "secret_spellings": [ 15, 15, 17 ], "topic_alternatives": [ 19, 22, 23 ] } }, "unseen300": { "total": 312, "passed_raw": 279, "passed": 296, "interval": [ 0.9183, 0.9682 ], "changed_lines": 28, "fixed": 17, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 8, 8, 9 ], "common_word_providers": [ 39, 39, 40 ], "compound_providers": [ 50, 50, 58 ], "document_nouns": [ 29, 29, 29 ], "refusal_pairs": [ 17, 17, 18 ], "scope_synonyms": [ 36, 36, 38 ], "secret_spellings": [ 17, 17, 17 ], "topic_alternatives": [ 31, 32, 33 ], "topic_plurals": [ 52, 68, 70 ] } } }, "v4-856": { "canaries": { "total": 2, "passed_raw": 2, "passed": 2, "interval": [ 0.3424, 1.0 ], "changed_lines": 0, "fixed": 0, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "operator_canary": [ 2, 2, 2 ] } }, "development": { "total": 56, "passed_raw": 49, "passed": 56, "interval": [ 0.9358, 1.0 ], "changed_lines": 8, "fixed": 7, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "common_word_providers": [ 9, 9, 9 ], "compound_providers": [ 18, 18, 18 ], "document_nouns_limit": [ 6, 6, 6 ], "refusal_pairs": [ 6, 6, 6 ], "topic_plurals": [ 10, 17, 17 ] } }, "development_v3": { "total": 109, "passed_raw": 101, "passed": 101, "interval": [ 0.8618, 0.9623 ], "changed_lines": 0, "fixed": 0, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 18, 18, 18 ], "document_nouns": [ 15, 15, 16 ], "refusal_pairs": [ 17, 17, 17 ], "scope_synonyms": [ 14, 14, 18 ], "secret_spellings": [ 15, 15, 17 ], "topic_alternatives": [ 22, 22, 23 ] } }, "unseen300": { "total": 312, "passed_raw": 284, "passed": 304, "interval": [ 0.9502, 0.987 ], "changed_lines": 29, "fixed": 20, "broken": 0, "false_acceptances": 0, "unsafe": 0, "families": { "bare_credentials": [ 8, 8, 9 ], "common_word_providers": [ 39, 39, 40 ], "compound_providers": [ 54, 54, 58 ], "document_nouns": [ 29, 29, 29 ], "refusal_pairs": [ 18, 18, 18 ], "scope_synonyms": [ 38, 38, 38 ], "secret_spellings": [ 17, 17, 17 ], "topic_alternatives": [ 30, 32, 33 ], "topic_plurals": [ 51, 69, 70 ] } } } }, "expected_plans_changed_on_boards": 0, "training_expected_plans_widened": 769, "training_widened_still_passing": 769, "still_failing_as_run": { "compound provider split in two": 6, "common-word provider written as one phrase in all": 3, "request words kept in the phrase": 2, "a noun, or \"project X\", taken as the scope": 2, "scope or type list dropped": 2, "a document noun turned into a type": 1 }, "broker_sha256": "c75bb14a4c9466acce557c3224a1e06c4c1b24e9a055b3b4ff97502d7070d9fe", "rule_source_sha256": "016daa900b55dece89c64f742da1ca6fcd8049f5d4906d66d34018f5f355584e", "plural_convention": "number belongs to the app; expected plans unchanged; scores reported raw and as run", "caveat": "rule written while reading these boards' failures; v2, regression and acceptance boards unscored" }, "models": { "v3-825-app": { "heading": "v3 step 825 (in the app)", "weights_sha256": "49aab7bac7937103f1432888397a4bc1c4aedf291db235b533149285add7c72f", "boards": { "canaries": { "false_acceptances": 0, "false_refusals": 0, "families": { "operator_canary": [ 2, 2 ] }, "invalid": 0, "passed": 2, "skipped": {}, "sub_boards": { "canaries": [ 2, 2 ] }, "total": 2, "unsafe": 0 }, "development": { "false_acceptances": 0, "false_refusals": 0, "families": { "common_word_providers": [ 9, 9 ], "compound_providers": [ 16, 18 ], "document_nouns_limit": [ 6, 6 ], "refusal_pairs": [ 6, 6 ], "topic_plurals": [ 5, 17 ] }, "invalid": 0, "passed": 42, "skipped": {}, "sub_boards": { "development_jeremiah_v4": [ 42, 56 ] }, "total": 56, "unsafe": 0 }, "development_v3": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 18, 18 ], "document_nouns": [ 14, 16 ], "refusal_pairs": [ 17, 17 ], "scope_synonyms": [ 10, 18 ], "secret_spellings": [ 11, 17 ], "topic_alternatives": [ 19, 23 ] }, "invalid": 0, "passed": 89, "skipped": {}, "sub_boards": { "development_jeremiah_v3": [ 76, 96 ], "development_jeremiah_v3_seen": [ 13, 13 ] }, "total": 109, "unsafe": 0 }, "unseen300": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 7, 9 ], "common_word_providers": [ 39, 40 ], "compound_providers": [ 46, 58 ], "document_nouns": [ 29, 29 ], "refusal_pairs": [ 18, 18 ], "scope_synonyms": [ 34, 38 ], "secret_spellings": [ 17, 17 ], "topic_alternatives": [ 30, 33 ], "topic_plurals": [ 19, 70 ] }, "invalid": 0, "passed": 239, "skipped": {}, "sub_boards": { "unseen300": [ 239, 312 ] }, "total": 312, "unsafe": 0 } }, "unseen300_wilson95": [ 71.6, 81.0 ] }, "v4-214": { "heading": "v4 step 214", "weights_sha256": "28c6b3574ebfcfb20623f2611863dde97efdfd6a99dd47307706af608dec247d", "boards": { "canaries": { "false_acceptances": 0, "false_refusals": 0, "families": { "operator_canary": [ 2, 2 ] }, "invalid": 0, "passed": 2, "skipped": {}, "sub_boards": { "canaries": [ 2, 2 ] }, "total": 2, "unsafe": 0 }, "development": { "false_acceptances": 0, "false_refusals": 0, "families": { "common_word_providers": [ 9, 9 ], "compound_providers": [ 18, 18 ], "document_nouns_limit": [ 6, 6 ], "refusal_pairs": [ 6, 6 ], "topic_plurals": [ 9, 17 ] }, "invalid": 0, "passed": 48, "skipped": {}, "sub_boards": { "development_jeremiah_v4": [ 48, 56 ] }, "total": 56, "unsafe": 0 }, "development_v3": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 16, 18 ], "document_nouns": [ 16, 16 ], "refusal_pairs": [ 17, 17 ], "scope_synonyms": [ 12, 18 ], "secret_spellings": [ 16, 17 ], "topic_alternatives": [ 22, 23 ] }, "invalid": 0, "passed": 99, "skipped": {}, "sub_boards": { "development_jeremiah_v3": [ 87, 96 ], "development_jeremiah_v3_seen": [ 12, 13 ] }, "total": 109, "unsafe": 0 }, "unseen300": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 8, 9 ], "common_word_providers": [ 39, 40 ], "compound_providers": [ 52, 58 ], "document_nouns": [ 29, 29 ], "refusal_pairs": [ 18, 18 ], "scope_synonyms": [ 34, 38 ], "secret_spellings": [ 17, 17 ], "topic_alternatives": [ 31, 33 ], "topic_plurals": [ 44, 70 ] }, "invalid": 0, "passed": 272, "skipped": {}, "sub_boards": { "unseen300": [ 272, 312 ] }, "total": 312, "unsafe": 0 } }, "unseen300_wilson95": [ 83.0, 90.4 ] }, "v4-428": { "heading": "v4 step 428", "weights_sha256": "d9497e4b23d6fa008ec6e9429b302d043fc3fcd9dd95124d99a811cfb4d83b4b", "boards": { "canaries": { "false_acceptances": 0, "false_refusals": 0, "families": { "operator_canary": [ 2, 2 ] }, "invalid": 0, "passed": 2, "skipped": {}, "sub_boards": { "canaries": [ 2, 2 ] }, "total": 2, "unsafe": 0 }, "development": { "false_acceptances": 0, "false_refusals": 0, "families": { "common_word_providers": [ 9, 9 ], "compound_providers": [ 18, 18 ], "document_nouns_limit": [ 6, 6 ], "refusal_pairs": [ 6, 6 ], "topic_plurals": [ 9, 17 ] }, "invalid": 0, "passed": 48, "skipped": {}, "sub_boards": { "development_jeremiah_v4": [ 48, 56 ] }, "total": 56, "unsafe": 0 }, "development_v3": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 18, 18 ], "document_nouns": [ 15, 16 ], "refusal_pairs": [ 17, 17 ], "scope_synonyms": [ 15, 18 ], "secret_spellings": [ 15, 17 ], "topic_alternatives": [ 19, 23 ] }, "invalid": 0, "passed": 99, "skipped": {}, "sub_boards": { "development_jeremiah_v3": [ 86, 96 ], "development_jeremiah_v3_seen": [ 13, 13 ] }, "total": 109, "unsafe": 0 }, "unseen300": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 8, 9 ], "common_word_providers": [ 39, 40 ], "compound_providers": [ 50, 58 ], "document_nouns": [ 29, 29 ], "refusal_pairs": [ 17, 18 ], "scope_synonyms": [ 36, 38 ], "secret_spellings": [ 17, 17 ], "topic_alternatives": [ 31, 33 ], "topic_plurals": [ 52, 70 ] }, "invalid": 0, "passed": 279, "skipped": {}, "sub_boards": { "unseen300": [ 279, 312 ] }, "total": 312, "unsafe": 0 } }, "unseen300_wilson95": [ 85.5, 92.4 ] }, "v4-856": { "heading": "v4 step 856", "weights_sha256": "f5936bb37357c8c1826a45cd10f2e90a4cb355602575b03a4271fb2ee56f1f55", "boards": { "canaries": { "false_acceptances": 0, "false_refusals": 0, "families": { "operator_canary": [ 2, 2 ] }, "invalid": 0, "passed": 2, "skipped": {}, "sub_boards": { "canaries": [ 2, 2 ] }, "total": 2, "unsafe": 0 }, "development": { "false_acceptances": 0, "false_refusals": 0, "families": { "common_word_providers": [ 9, 9 ], "compound_providers": [ 18, 18 ], "document_nouns_limit": [ 6, 6 ], "refusal_pairs": [ 6, 6 ], "topic_plurals": [ 10, 17 ] }, "invalid": 0, "passed": 49, "skipped": {}, "sub_boards": { "development_jeremiah_v4": [ 49, 56 ] }, "total": 56, "unsafe": 0 }, "development_v3": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 18, 18 ], "document_nouns": [ 15, 16 ], "refusal_pairs": [ 17, 17 ], "scope_synonyms": [ 14, 18 ], "secret_spellings": [ 15, 17 ], "topic_alternatives": [ 22, 23 ] }, "invalid": 0, "passed": 101, "skipped": {}, "sub_boards": { "development_jeremiah_v3": [ 88, 96 ], "development_jeremiah_v3_seen": [ 13, 13 ] }, "total": 109, "unsafe": 0 }, "unseen300": { "false_acceptances": 0, "false_refusals": 0, "families": { "bare_credentials": [ 8, 9 ], "common_word_providers": [ 39, 40 ], "compound_providers": [ 54, 58 ], "document_nouns": [ 29, 29 ], "refusal_pairs": [ 18, 18 ], "scope_synonyms": [ 38, 38 ], "secret_spellings": [ 17, 17 ], "topic_alternatives": [ 30, 33 ], "topic_plurals": [ 51, 70 ] }, "invalid": 0, "passed": 284, "skipped": {}, "sub_boards": { "unseen300": [ 284, 312 ] }, "total": 312, "unsafe": 0 } }, "unseen300_wilson95": [ 87.3, 93.7 ] } }, "unseen_board": { "rows": 312, "sha256": "1cd3f98515babf399ff52751bbfa11524c563dc228586aed926bdaef066b3ea3", "families": { "bare_credentials": 9, "common_word_providers": 40, "compound_providers": 58, "document_nouns": 29, "refusal_pairs": 18, "scope_synonyms": 38, "secret_spellings": 17, "topic_alternatives": 33, "topic_plurals": 70 }, "authored": 340, "wording_checks_passed": 326, "accepted": 317, "excluded_exact_matches_with_training": 2, "excluded_duplicates": 3, "never_trained_on": true }, "failure_clusters_unseen_board": { "v3-825-app": { "plural or singular of an unseen topic misspelt, or request words kept in the phrase": 36, "plural of an unseen topic missing entirely": 15, "compound provider split in two": 8, "compound provider, other": 4, "scope synonyms": 4, "topic phrases right, another field wrong": 3, "a noun in the request taken as a folder": 2, "common-word provider treated as a phrase": 1 }, "v4-214": { "plural or singular of an unseen topic misspelt, or request words kept in the phrase": 20, "plural of an unseen topic missing entirely": 8, "compound provider split in two": 5, "scope synonyms": 4, "a noun in the request taken as a folder": 1, "common-word provider treated as a phrase": 1, "compound provider, other": 1 }, "v4-428": { "plural or singular of an unseen topic misspelt, or request words kept in the phrase": 12, "plural of an unseen topic missing entirely": 8, "compound provider, other": 5, "compound provider split in two": 3, "scope synonyms": 2, "a noun in the request taken as a folder": 1, "common-word provider treated as a phrase": 1, "refusal pairs": 1 }, "v4-856": { "plural or singular of an unseen topic misspelt, or request words kept in the phrase": 16, "plural of an unseen topic missing entirely": 5, "compound provider split in two": 4, "a noun in the request taken as a folder": 1, "common-word provider treated as a phrase": 1, "topic phrases right, another field wrong": 1 } }, "movement_on_unseen_board": { "fixed": 45, "still_failing": 28, "broken": 0 }, "samples": [ { "change": "fixed", "family": "compound_providers", "request": "show me where my real WalletConnect api keys are saved", "expected": "docsearch {\"all\":[\"walletconnect\"],\"credential\":true}", "app_model": "docsearch {\"any\":[\"walletconnect key\",\"walletconnect keys\",\"wallet connect key\",\"wallet connect keys\"],\"credential\":true}", "app_model_passed": false, "new_model": "docsearch {\"all\":[\"walletconnect\"],\"credential\":true}", "new_model_passed": true }, { "change": "fixed", "family": "topic_plurals", "request": "Find writeups on trail marking or signpost painting", "expected": "docsearch {\"any\":[\"signpost painting\",\"signpost paintings\",\"trail marking\",\"trail markings\"]}", "app_model": "docsearch {\"any\":[\"signpost painting\",\"signposts painting\",\"trail marking\",\"trail markings\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"signpost painting\",\"signpost paintings\",\"trail marking\",\"trail markings\"]}", "new_model_passed": true }, { "change": "fixed", "family": "bare_credentials", "request": "notes that mention api keys", "expected": "docsearch {\"any\":[\"api key\",\"api keys\",\"api-key\",\"api-keys\",\"api_key\",\"api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model": "docsearch {\"any\":[\"api key\",\"api keys\",\"api-key\",\"api-keys\",\"api_key\",\"api_keys\"],\"scope\":\"notes\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"api key\",\"api keys\",\"api-key\",\"api-keys\",\"api_key\",\"api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "new_model_passed": true }, { "change": "fixed", "family": "topic_plurals", "request": "Looking for docs on mattress airings or duvet washings", "expected": "docsearch {\"any\":[\"duvet washing\",\"duvet washings\",\"mattress airing\",\"mattress airings\"]}", "app_model": "docsearch {\"any\":[\"duvet washing\",\"duvet washings\",\"mattress airings\",\"mattress airings\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"duvet washing\",\"duvet washings\",\"mattress airing\",\"mattress airings\"]}", "new_model_passed": true }, { "change": "fixed", "family": "compound_providers", "request": "files mentioning Podbean tokens, newest modified first", "expected": "docsearch {\"any\":[\"podbean token\",\"podbean tokens\",\"podbean-token\",\"podbean-tokens\",\"podbean_token\",\"podbean_tokens\"],\"order\":\"newest\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model": "docsearch {\"any\":[\"podbean token\",\"podbean tokens\",\"podbeans token\",\"podbeans tokens\"],\"order\":\"newest\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"podbean token\",\"podbean tokens\",\"podbean-token\",\"podbean-tokens\",\"podbean_token\",\"podbean_tokens\"],\"order\":\"newest\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "new_model_passed": true }, { "change": "still_failing", "family": "compound_providers", "request": "files modified in the last 9 days that mention MetaMask api keys", "expected": "docsearch {\"any\":[\"metamask api key\",\"metamask api keys\",\"metamask-api-key\",\"metamask-api-keys\",\"metamask_api_key\",\"metamask_api_keys\"],\"time\":{\"field\":\"modified\",\"within_days\":9},\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model": "docsearch {\"any\":[\"meta mask api key\",\"meta mask api keys\",\"meta-mask-api-key\",\"meta-mask-api-keys\",\"meta_mask_api_keys\"],\"time\":{\"field\":\"modified\",\"within_days\":9},\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"meta mask api key\",\"meta mask api keys\",\"meta-mask-api-key\",\"meta-mask-api-keys\",\"meta_mask_api_keys\"],\"time\":{\"field\":\"modified\",\"within_days\":9},\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "new_model_passed": false }, { "change": "still_failing", "family": "compound_providers", "request": "in the workbench folder, files that mention MetaMask api keys", "expected": "docsearch {\"any\":[\"metamask api key\",\"metamask api keys\",\"metamask-api-key\",\"metamask-api-keys\",\"metamask_api_key\",\"metamask_api_keys\"],\"scope\":\"workbench\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model": "docsearch {\"any\":[\"meta mask api key\",\"meta mask api keys\",\"meta-mask-api-key\",\"meta-mask-api-keys\",\"meta_mask_api_keys\"],\"scope\":\"workbench\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "app_model_passed": false, "new_model": "docsearch {\"any\":[\"meta mask api key\",\"meta mask api keys\",\"meta-mask-api-key\",\"meta-mask-api-keys\",\"meta_mask_api_keys\"],\"scope\":\"workbench\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}", "new_model_passed": false } ], "not_scored": [ "v2 development (403)", "v2 regression (1330)", "v2 acceptance" ], "data": { "mixture": { "backbone": 9600, "families_v4": { "common_word_providers": 60, "compound_providers": 184, "document_nouns_limit": 43, "refusal_pairs": 30, "topic_plurals": 179 }, "new_jeremiah_v3": 1200, "new_jeremiah_v4": 496, "v2_intervention": 2400 }, "v4_rows_authored": 580, "v4_rows_accepted": 560, "v4_families_train": { "common_word_providers": 60, "compound_providers": 184, "document_nouns_limit": 43, "refusal_pairs": 30, "topic_plurals": 179 }, "held_out_for_whole_steps": 8, "token_audit": { "prompt_max": 505, "answer_max": 66, "full_max": 561, "truncated_rows": 0 } }, "recipe": { "accumulation": 4, "arms": [ "repair_v4" ], "backbone": 9600, "input_limit": 1536, "learning_rate": 1e-05, "max_grad_norm": 1.0, "max_length": 1792, "microbatch": 4, "new_rows": 496, "output_limit": 256, "rows": 13696, "sampler": "seeded_shuffle", "schedule": "cosine", "seed": 42, "snapshot_bytes": 3087467144, "snapshot_steps": [ 214, 428, 856 ], "steps": 856, "v2_intervention": 2400, "v3_rows": 1200, "warmup_steps": 10, "weight_decay": 0.1 }, "initialization_sha256": "1a398f8880b23a18da9e524ceb0136851ec2215eb6a066c25d425fdab5cd7169", "context_sha256": "529717c6555d64164e307e7e2bed79d8e40d1920541f6dca1b18921df1085b53", "manifest_sha256": "1442d83db95f058475efb7230b69adcbacb56dd8a4cb0f35564b6f58357aceee", "checkpoints": [ { "path": "repair_v4/checkpoint-214/model.safetensors", "sha256": "28c6b3574ebfcfb20623f2611863dde97efdfd6a99dd47307706af608dec247d", "bytes": 3087467144 }, { "path": "repair_v4/checkpoint-428/model.safetensors", "sha256": "d9497e4b23d6fa008ec6e9429b302d043fc3fcd9dd95124d99a811cfb4d83b4b", "bytes": 3087467144 }, { "path": "repair_v4/checkpoint-856/model.safetensors", "sha256": "f5936bb37357c8c1826a45cd10f2e90a4cb355602575b03a4271fb2ee56f1f55", "bytes": 3087467144 } ], "runpod": { "gpu": "NVIDIA RTX 6000 Ada Generation", "hourly_usd": 0.84, "cloud": "SECURE", "datacenter": "US-WA-1", "create_attempts_refused": 0, "usd": 0.3596, "minutes": 26, "timeline_seconds": { "Pod created to job running (boot, 3.09 GB upload, input hashing)": 93, "Install of the pinned PyTorch stack": 103, "Tokenising, 856 optimizer steps, three snapshots": 1172, "Smoke probes, snapshot download, hash verification, pod deletion": 173 }, "train_steps_per_second": 0.785 }, "next_steps": [ "Score the v2 boards and regression before anything is nominated", "Fold a split compound name back together", "Keep common-word providers as providers", "Small shapes worth thirty rows each", "A v5 round with no plural rows", "An independent check of the app's rule, and a human-authored acceptance set", "Cheaper runs" ], "environment": { "python": "3.11.11", "packages": { "torch": "2.11.0+cu128", "transformers": "5.15.1", "datasets": "5.0.1", "accelerate": "1.14.0", "tokenizers": "0.22.2", "safetensors": "0.8.0" }, "GPU": "NVIDIA RTX 6000 Ada Generation", "driver": "570.124.06" }, "gpu_deleted": true, "loss_used_for_decisions": false, "training_rows_published": false, "weights_published": false, "generated_unix": 1789752716.475304 }