commit 7fe4e678a5f6c9067b5f2e43644058ae2ee76969 Author: Chen, Li Date: Sat May 9 15:53:43 2026 +0800 doris init diff --git a/Doris.md b/Doris.md new file mode 100644 index 0000000..a887c30 --- /dev/null +++ b/Doris.md @@ -0,0 +1,26 @@ +### Doris + +因VM CPU不支持AVX2指令集，需要构建最新版本（4.1）Doris-noavx2镜像，此版本对比正常版本性能会低一些 + +#### VM信息 +| VM | Service | Port | +| :---: | :---: | --- | +| 6.86.80.12 | Doris FE | 9030（msql port）8030（http port） | +| 6.86.80.13 | Doris BE | 9050（health） | + +#### Resource配置 +创建AI Resource语句，以下Resource已创建好，相关函数使用参考 [Doris官网](https://doris.apache.org/zh-CN/docs/4.x/ai/ai-overview) +``` +CREATE RESOURCE 'deepseek_v3' +PROPERTIES ( + 'type' = 'ai', + 'ai.provider_type' = 'openai', + 'ai.endpoint' = 'http://6.86.80.4:30080/v1/chat/completions', + 'ai.model_name' = 'deepseek-v3.2', + 'ai.temperature' = '0.7', + 'ai.max_retries' = '3', + 'ai.api_key' = 'sk-5HeY7gfSIlyZMacfuXOf5cphpymsNqufEu1ou4U3avbULcyY' +); + +SELECT AI_GENERATE('deepseek_v3', 'Hello') AS Result; +``` \ No newline at end of file diff --git a/be/Dockerfile b/be/Dockerfile new file mode 100644 index 0000000..cb026a4 --- /dev/null +++ b/be/Dockerfile @@ -0,0 +1,18 @@ +FROM eclipse-temurin:17.0.18_8-jre-ubi9-minimal + +ENV JAVA_HOME="/opt/java/openjdk" +ENV PATH="/opt/apache-doris/be/bin:$PATH" +ENV TZ="Asia/Shanghai" + +RUN mkdir -p /opt/apache-doris + +RUN microdnf install -y util-linux && \ + microdnf clean all + +COPY apache-doris-4.1.0-bin-x64-noavx2/be /opt/apache-doris/be + +RUN chmod 755 /opt/apache-doris/be/bin/start_be.sh + +EXPOSE 9060 9050 8060 8040 + +ENTRYPOINT ["/opt/apache-doris/be/bin/start_be.sh", "--console"] \ No newline at end of file diff --git a/be/be.conf b/be/be.conf new file mode 100644 index 0000000..a4e8edb --- /dev/null +++ b/be/be.conf @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CUR_DATE=`date +%Y%m%d-%H%M%S` + +# Log dir +LOG_DIR="${DORIS_HOME}/log/" + +# For jdk 17, this JAVA_OPTS will be used as default JVM options +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" +# Set your own JAVA_HOME +# JAVA_HOME=/path/to/jdk/ + +# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile +# https://jemalloc.net/jemalloc.3.html +JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:true,prof_active:false,lg_prof_interval:-1,lg_extent_max_active_fit:8" +JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_" + +# ports for admin, web, heartbeat service +be_port = 9060 +webserver_port = 8040 +heartbeat_service_port = 9050 +brpc_port = 8060 +arrow_flight_sql_port = 8050 + +# HTTPS configures +enable_https = false +# path of certificate in PEM format. +ssl_certificate_path = "$DORIS_HOME/conf/cert.pem" +# path of private key in PEM format. +ssl_private_key_path = "$DORIS_HOME/conf/key.pem" + +# Choose one if there are more than one ip except loopback address. +# Note that there should at most one ip match this list. +# If no ip match this rule, will choose one randomly. +# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 +# Default value is empty. +priority_networks = 6.86.80.0/24 + +# data root path, separate by ';' +# You can specify the storage type for each root path, HDD (cold data) or SSD (hot data) +# eg: +# storage_root_path = /home/disk1/doris;/home/disk2/doris;/home/disk2/doris +# storage_root_path = /home/disk1/doris,medium:SSD;/home/disk2/doris,medium:SSD;/home/disk2/doris,medium:HDD +# /home/disk2/doris,medium:HDD(default) +# +# you also can specify the properties by setting ':', separate by ',' +# property 'medium' has a higher priority than the extension of path +# +# Default value is ${DORIS_HOME}/storage, you should create it by hand. +storage_root_path = ${DORIS_HOME}/storage,medium:HDD + +# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers +# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers + +# Advanced configurations +# INFO, WARNING, ERROR, FATAL +sys_log_level = INFO +# sys_log_roll_mode = SIZE-MB-1024 +# sys_log_roll_num = 10 +# sys_log_verbose_modules = * +# log_buffer_level = -1 + +# aws sdk log level +# Off = 0, +# Fatal = 1, +# Error = 2, +# Warn = 3, +# Info = 4, +# Debug = 5, +# Trace = 6 +# Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs +aws_log_level = 2 + +# azure sdk log level +# Verbose = 1, +# Informational = 2, +# Warning = 3, +# Error = 4 +azure_log_level = 4 +## If you are not running in aws cloud, you can disable EC2 metadata +AWS_EC2_METADATA_DISABLED=true diff --git a/be/docker-compose.yml b/be/docker-compose.yml new file mode 100644 index 0000000..2693053 --- /dev/null +++ b/be/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.8' + +services: + doris-be: + image: doris-be:4.1 + container_name: doris-be + restart: always + security_opt: + - label=disable + ulimits: + nofile: + soft: 655350 + hard: 655350 + environment: + - TZ=Asia/Shanghai + - DORIS_HOME=/opt/apache-doris/be + network_mode: host + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "/home/tscnlich/doris/storage:/opt/apache-doris/be/storage/" + - "/home/tscnlich/doris/logs:/opt/apache-doris/be/log/" + - "/home/tscnlich/doris/be.conf:/opt/apache-doris/be/conf/be.conf" diff --git a/fe/Dockerfile b/fe/Dockerfile new file mode 100644 index 0000000..80c1dc0 --- /dev/null +++ b/fe/Dockerfile @@ -0,0 +1,17 @@ +FROM eclipse-temurin:17.0.18_8-jre-ubi9-minimal + +ENV JAVA_HOME="/opt/java/openjdk" +ENV PATH="/opt/apache-doris/fe/bin:$PATH" +ENV TZ="Asia/Shanghai" + +RUN mkdir -p /opt/apache-doris && \ + microdnf install -y mariadb util-linux && \ + microdnf clean all + +COPY apache-doris-4.1.0-bin-x64-noavx2/fe /opt/apache-doris/fe + +RUN chmod 755 /opt/apache-doris/fe/bin/start_fe.sh + +EXPOSE 8030 9010 9020 9030 + +ENTRYPOINT ["/opt/apache-doris/fe/bin/start_fe.sh", "--console"] \ No newline at end of file diff --git a/fe/docker-compose.yml b/fe/docker-compose.yml new file mode 100644 index 0000000..c303e60 --- /dev/null +++ b/fe/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.8' + +services: + doris-fe: + image: doris-fe:4.1 + container_name: doris-fe + restart: always + environment: + - TZ=Asia/Shanghai + - DORIS_HOME=/opt/apache-doris/fe + network_mode: host + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "/home/tscnlich/doris/meta:/opt/apache-doris/fe/doris-meta/" + - "/home/tscnlich/doris/logs:/opt/apache-doris/fe/log/" + - "/home/tscnlich/doris/fe.conf:/opt/apache-doris/fe/conf/fe.conf" \ No newline at end of file diff --git a/fe/fe.conf b/fe/fe.conf new file mode 100644 index 0000000..7ee6154 --- /dev/null +++ b/fe/fe.conf @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +##################################################################### +## The uppercase properties are read and exported by bin/start_fe.sh. +## To see all Frontend configurations, +## see fe/src/org/apache/doris/common/Config.java +##################################################################### + +CUR_DATE=`date +%Y%m%d-%H%M%S` + +# Log dir +LOG_DIR = ${DORIS_HOME}/log + +# For jdk 17, this JAVA_OPTS will be used as default JVM options +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" + +# Set your own JAVA_HOME +# JAVA_HOME=/path/to/jdk/ + +## +## the lowercase properties are read by main program. +## + +# store metadata, must be created before start FE. +# Default value is ${DORIS_HOME}/doris-meta +# meta_dir = ${DORIS_HOME}/doris-meta + +# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers +# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers + +http_port = 8030 +rpc_port = 9020 +query_port = 9030 +edit_log_port = 9010 +arrow_flight_sql_port = 8070 + +# Choose one if there are more than one ip except loopback address. +# Note that there should at most one ip match this list. +# If no ip match this rule, will choose one randomly. +# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 +# Default value is empty. +priority_networks = 6.86.80.0/24 + +# Advanced configurations +# log_roll_size_mb = 1024 +# INFO, WARN, ERROR, FATAL +sys_log_level = INFO +# NORMAL, BRIEF, ASYNC +sys_log_mode = ASYNC +# sys_log_roll_num = 10 +# sys_log_verbose_modules = org.apache.doris +# audit_log_dir = $LOG_DIR +# audit_log_modules = slow_query, query +# audit_log_roll_num = 10 +# meta_delay_toleration_second = 10 +# qe_max_connection = 1024 +# qe_query_timeout_second = 300 +# qe_slow_log_ms = 5000