{
  "model": [
    {
      "name": "model name",
      "model_class": "HuggingFaceCausalLM",
      "parameters": {
        "path": "path to model",
        "model_max_length": 4096,
        "tokenizer_path": "",
        "tokenizer_kwargs": {
          "trust_remote_code": true
        },
        "peft_path": null,
        "model_kwargs": {
          "torch_dtype": "torch.float32",
          "trust_remote_code": true
        },
        "prompt_template": "plain",
        "batch_size": 4
      }
    },
    {
      "name": "model2 name",
      "model_class": "HuggingFaceCausalLM",
      "parameters": {
        "path": "path to model2",
        "model_max_length": 4096,
        "tokenizer_path": "",
        "tokenizer_kwargs": {
          "trust_remote_code": true
        },
        "peft_path": null,
        "model_kwargs": {
          "torch_dtype": "torch.float32",
          "trust_remote_code": true
        },
        "prompt_template": "plain",
        "batch_size": 4
      }
    }
  ],
  "dataset": [
    {
      "name": "agieval",
      "dataset_class": "AGIEvalDataset",
      "debug": false,
      "few_shot": false,
      "path": "path to original dataset (folder)",
      "save_path": "path to save converted dataset (e.g. inference_data/agieval.json)"
    },
    {
      "name": "ceval",
      "dataset_class": "CEvalDataset",
      "debug": false,
      "few_shot": true,
      "path": "path to original dataset (folder)",
      "save_path": "path to save converted dataset (e.g. inference_data/ceval.json)"
    },
    {
      "name": "cmmlu",
      "dataset_class": "CMMLUDataset",
      "debug": false,
      "few_shot": true,
      "path": "path to original dataset (folder)",
      "save_path": "path to save converted dataset (e.g. inference_data/cmmlu.json)"
    },
    {
      "name": "gaokaobench",
      "dataset_class": "GaoKaoBenchDataset",
      "debug": false,
      "few_shot": false,
      "path": "path to original dataset (folder)",
      "save_path": "path to save converted dataset (e.g. inference_data/gaokaobench.json)"
    },
    {
      "name": "mmlu",
      "dataset_class": "MMLUDataset",
      "debug": false,
      "few_shot": true,
      "path": "path to original dataset (folder)",
      "save_path": "path to save converted dataset (e.g. inference_data/mmlu.json)"
    }
  ]
}
