rag-experiment-accelerator/02_qa_generation.py at development · microsoft/rag-experiment-accelerator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
import argparse

from rag_experiment_accelerator.checkpoint import init_checkpoint
from rag_experiment_accelerator.run.qa_generation import run
from rag_experiment_accelerator.config.config import Config
from rag_experiment_accelerator.config.environment import Environment
from rag_experiment_accelerator.config.paths import get_all_file_paths


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--config_path", type=str, help="input: path to the config file"
    )
    parser.add_argument("--data_dir", type=str, help="input: path to the input data")
    args, _ = parser.parse_known_args()

    environment = Environment.from_env_or_keyvault()
    config = Config.from_path(environment, args.config_path, args.data_dir)
    init_checkpoint(config)

    run(environment, config, get_all_file_paths(config.path.data_dir))