109 lines
4.1 KiB
Plaintext
109 lines
4.1 KiB
Plaintext
|
# Main data configuration (array format)
|
||
|
|
- data_path: "https://your-blob-storage.blob.core.windows.net/container?sas-token"
|
||
|
|
datasource_name: "CATOnline-cn" # data source name
|
||
|
|
data_dir: "" # Optional local data directory
|
||
|
|
base_path: "/app/run_tmp" # Temporary processing directory
|
||
|
|
|
||
|
|
# File processing limits
|
||
|
|
process_file_num: 0 # 0 = process all files
|
||
|
|
process_file_last_modify: "2025-06-24 00:00:00" # Only process files modified after this date
|
||
|
|
|
||
|
|
# Chunking configuration
|
||
|
|
chunk_size: 2048 # Maximum tokens per chunk
|
||
|
|
token_overlap: 128 # Overlap between chunks
|
||
|
|
|
||
|
|
# Index schemas configuration
|
||
|
|
index_schemas:
|
||
|
|
# Chunk-level index for search
|
||
|
|
- index_name: "your-knowledge-chunk-index"
|
||
|
|
data_type: ["metadata", "document", "chunk"]
|
||
|
|
field_type: "append" # How to handle existing data
|
||
|
|
upload_batch_size: 50 # Documents per batch upload
|
||
|
|
|
||
|
|
# Metadata fields to include
|
||
|
|
fields: [
|
||
|
|
"filepath", "timestamp", "title", "publisher", "publish_date",
|
||
|
|
"document_category", "document_code", "language_code",
|
||
|
|
"x_Standard_Regulation_Id", "x_Attachment_Type",
|
||
|
|
"x_Standard_Title_CN", "x_Standard_Title_EN",
|
||
|
|
"x_Standard_Published_State", "x_Standard_Drafting_Status",
|
||
|
|
"x_Standard_Range", "x_Standard_Kind", "x_Standard_No",
|
||
|
|
"x_Standard_Code", "x_Standard_Technical_Committee",
|
||
|
|
"x_Standard_Vehicle_Type", "x_Standard_Power_Type",
|
||
|
|
"x_Standard_CCS", "x_Standard_ICS",
|
||
|
|
"x_Standard_Published_Date", "x_Standard_Effective_Date",
|
||
|
|
"x_Regulation_Status", "x_Regulation_Title_CN",
|
||
|
|
"x_Regulation_Title_EN", "x_Regulation_Document_No",
|
||
|
|
"x_Regulation_Issued_Date", "x_Classification",
|
||
|
|
"x_Work_Group", "x_Reference_Standard",
|
||
|
|
"x_Replaced_by", "x_Refer_To", "func_uuid",
|
||
|
|
"update_time", "status"
|
||
|
|
]
|
||
|
|
|
||
|
|
# Vector configuration
|
||
|
|
vector_fields:
|
||
|
|
- field: "contentVector"
|
||
|
|
append_fields: ["content"] # Fields to vectorize for content
|
||
|
|
- field: "full_metadata_vector"
|
||
|
|
append_fields: ["full_headers", "doc_metadata"] # Metadata vectorization
|
||
|
|
|
||
|
|
# Azure AI Search configuration
|
||
|
|
semantic_config_name: "default"
|
||
|
|
vector_config_name: "vectorSearchProfile"
|
||
|
|
update_by_field: "filepath" # Field to use for updates
|
||
|
|
full_metadata_vector_fields: ["full_headers", "doc_metadata"]
|
||
|
|
|
||
|
|
# Document-level index
|
||
|
|
- index_name: "your-knowledge-document-index"
|
||
|
|
data_type: ["document", "metadata"]
|
||
|
|
field_type: "full" # Replace entire documents
|
||
|
|
key_fields: ["filepath"] # Primary key fields
|
||
|
|
upload_batch_size: 1
|
||
|
|
|
||
|
|
fields: [
|
||
|
|
# Same field list as chunk index
|
||
|
|
"filepath", "timestamp", "title", "publisher"
|
||
|
|
# ... (same as above)
|
||
|
|
]
|
||
|
|
|
||
|
|
merge_content_fields: ["content"] # Fields to merge from chunks
|
||
|
|
vector_fields:
|
||
|
|
- field: "full_metadata_vector"
|
||
|
|
append_fields: ["doc_metadata"]
|
||
|
|
|
||
|
|
semantic_config_name: "default"
|
||
|
|
vector_config_name: "vectorSearchProfile"
|
||
|
|
update_by_field: "filepath"
|
||
|
|
|
||
|
|
# Regulation-specific index
|
||
|
|
- index_name: "your-regulation-index"
|
||
|
|
data_type: ["metadata"]
|
||
|
|
field_type: "full"
|
||
|
|
key_fields: ["x_Standard_Regulation_Id"] # Regulation ID as key
|
||
|
|
upload_batch_size: 50
|
||
|
|
|
||
|
|
fields: [
|
||
|
|
# Regulation-specific fields
|
||
|
|
"x_Standard_Regulation_Id", "x_Standard_Title_CN",
|
||
|
|
"x_Standard_Title_EN", "x_Regulation_Status"
|
||
|
|
# ... (regulation metadata fields)
|
||
|
|
]
|
||
|
|
|
||
|
|
vector_fields:
|
||
|
|
- field: "full_metadata_vector"
|
||
|
|
append_fields: ["doc_metadata"]
|
||
|
|
|
||
|
|
update_by_field: "x_Standard_Regulation_Id"
|
||
|
|
|
||
|
|
# Field merging configuration
|
||
|
|
merge_fields:
|
||
|
|
- key: "doc_metadata" # Combined metadata field
|
||
|
|
fields: [
|
||
|
|
"title", "publisher", "document_category", "document_code",
|
||
|
|
"x_Standard_Title_CN", "x_Standard_Title_EN",
|
||
|
|
"x_Standard_Published_State", "x_Standard_Drafting_Status"
|
||
|
|
# ... (all metadata fields to combine)
|
||
|
|
]
|
||
|
|
|
||
|
|
# Vector field configuration
|
||
|
|
full_metadata_vector_fields: ["full_headers", "doc_metadata"]
|