docs2vecs/docs/readme/sample-config-file-2.yml at main · AmadeusITGroup/docs2vecs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
definitions:
  - skill: &ScrollWordExporter
      type: exporter
      name: scrollword-exporter
      params:
        api_url: https://scroll-word.us.exporter.k15t.app/api/public/1/exports
        auth_token: env.SWE_AUTH_TOKEN
        poll_interval: 2
        export_folder: ~/Downloads/sw_export_temp
        scope: descendants
        page_urls:
            - https://confluence-page-url
        confluence_prefix: https://amadeus.atlassian.net

  - skill: &FileScanner
      type: file-scanner
      name: multi-file-scanner
      params:
        path: user/directory
        filter: ["*.txt"]
        recursive: true

  - skill: &FileReader
      type: file-reader
      name: multi-file-reader

  - skill: &TextSplitter
      type: splitter
      name: recursive-character-splitter
      params:
        chunk_size: 1000
        overlap: 100

  - skill: &SemanticSplitter
      type: splitter
      name: semantic-splitter
      params:
        embedding_model:
          endpoint: https://resource-name.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings?api-version=2023-05-15
          api_key: env.AZURE_EMBEDDING_API_KEY
          api_version: "2023-05-15"
          deployment_name: text-embedding-ada-002

  - skill: &AzureAda002
      type: embedding
      name: azure-ada002-embedding
      params:
        endpoint: https://resource-name.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings?api-version=2023-05-15
        api_key: env.AZURE_EMBEDDING_API_KEY
        api_version: "2023-05-15"
        deployment_name: text-embedding-ada-002

  - skill: &AzureAISearchId
      type: vector-store
      name: azure-ai-search
      params:
        api_key: env.AZURE_AI_SEARCH_API_KEY
        endpoint: https://resource-name.search.windows.net
        index_name: toolchain-index
        field_mapping:
          document_id: document_id
          content: content
          source_link: source_link
          document_name: document_name
          embedding: embedding

  - skillset: &TestSkillset1
      - *ScrollWordExporter
      - *FileReader
      - *SemanticSplitter
      - *AzureAda002
      - *AzureAISearchId

  - skillset: &TestSkillset2
      - *FileScanner
      - *FileReader
      - *SemanticSplitter
      - *AzureAda002
      - *AzureAISearchId

indexer:
    id: AzureAISearchIndexer
    skillset: *TestSkillset1