{
  "base_model": "Qwen/Qwen2.5-Coder-1.5B-Instruct",
  "sft_model": "Qwen2.5-Coder-1.5B-Instruct-sft-137row-e1",
  "n": 500,
  "device": "cuda",
  "dtype": "torch.bfloat16",
  "overall": {
    "base": {
      "has_fence": 0.998,
      "style": 0.0,
      "exact": 0.122,
      "first_line": 0.124,
      "stem": 0.504,
      "tool": 0.864,
      "empty": 0.0,
      "bash_f1": 0.41469812816692125,
      "text_f1": 0.414286132174385,
      "tf_loss": 1.5155438561588526,
      "tf_acc": 0.696757588326931,
      "n": 500
    },
    "sft": {
      "has_fence": 1.0,
      "style": 1.0,
      "exact": 0.14,
      "first_line": 0.14,
      "stem": 0.568,
      "tool": 0.916,
      "empty": 0.0,
      "bash_f1": 0.4382818742145525,
      "text_f1": 0.6633540410911339,
      "tf_loss": 1.2692855091183446,
      "tf_acc": 0.7628066260814667,
      "n": 500
    }
  },
  "by_kind": {
    "base": {
      "task_solution": {
        "has_fence": 0.998,
        "style": 0.0,
        "exact": 0.122,
        "first_line": 0.124,
        "stem": 0.504,
        "tool": 0.864,
        "empty": 0.0,
        "bash_f1": 0.41469812816692125,
        "text_f1": 0.414286132174385,
        "tf_loss": 1.5155438561588526,
        "tf_acc": 0.696757588326931,
        "n": 500
      }
    },
    "sft": {
      "task_solution": {
        "has_fence": 1.0,
        "style": 1.0,
        "exact": 0.14,
        "first_line": 0.14,
        "stem": 0.568,
        "tool": 0.916,
        "empty": 0.0,
        "bash_f1": 0.4382818742145525,
        "text_f1": 0.6633540410911339,
        "tf_loss": 1.2692855091183446,
        "tf_acc": 0.7628066260814667,
        "n": 500
      }
    }
  },
  "by_cloud": {
    "base": {
      "cloud": {
        "has_fence": 1.0,
        "style": 0.0,
        "exact": 0.0,
        "first_line": 0.0,
        "stem": 0.5555555555555556,
        "tool": 0.5555555555555556,
        "empty": 0.0,
        "bash_f1": 0.18255614211496562,
        "text_f1": 0.2472764473479802,
        "tf_loss": 1.7774598730934992,
        "tf_acc": 0.6507930722501543,
        "n": 9
      },
      "other": {
        "has_fence": 0.9979633401221996,
        "style": 0.0,
        "exact": 0.12423625254582485,
        "first_line": 0.12627291242362526,
        "stem": 0.5030549898167006,
        "tool": 0.869653767820774,
        "empty": 0.0,
        "bash_f1": 0.41895327658742554,
        "text_f1": 0.4173474094929952,
        "tf_loss": 1.5107429515714559,
        "tf_acc": 0.6976001150981957,
        "n": 491
      }
    },
    "sft": {
      "cloud": {
        "has_fence": 1.0,
        "style": 1.0,
        "exact": 0.0,
        "first_line": 0.0,
        "stem": 0.5555555555555556,
        "tool": 0.5555555555555556,
        "empty": 0.0,
        "bash_f1": 0.19795757295757296,
        "text_f1": 0.49497784567305303,
        "tf_loss": 1.4817757738961115,
        "tf_acc": 0.7136842012405396,
        "n": 9
      },
      "other": {
        "has_fence": 1.0,
        "style": 1.0,
        "exact": 0.1425661914460285,
        "first_line": 0.1425661914460285,
        "stem": 0.5682281059063137,
        "tool": 0.9226069246435845,
        "empty": 0.0,
        "bash_f1": 0.44268700397282706,
        "text_f1": 0.6664403664653961,
        "tf_loss": 1.26539057554808,
        "tf_acc": 0.7637070371274307,
        "n": 491
      }
    }
  }
}