diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index affa3049bce2..d990545520fd 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -25,6 +25,7 @@ extern "C" { #include "function.h" #include "scalar.h" + typedef struct SOperatorValueType { int32_t opResType; int32_t selfType; diff --git a/test/ci/filter_for_return_values b/test/ci/filter_for_return_values index c21ec391cedb..784cc06e5225 100644 --- a/test/ci/filter_for_return_values +++ b/test/ci/filter_for_return_values @@ -1,4 +1,5 @@ match callExpr( + isExpansionInMainFile(), hasParent(anyOf( compoundStmt(), doStmt(hasCondition(expr().bind("cond"))) @@ -9,28 +10,33 @@ match callExpr( hasName("tFreeTSmaWrapper") )))), unless(anyOf( - callee(functionDecl(hasName("memcpy"))), - callee(functionDecl(hasName("strcpy"))), - callee(functionDecl(hasName("strcat"))), - callee(functionDecl(hasName("strncpy"))), - callee(functionDecl(hasName("memset"))), - callee(functionDecl(hasName("memmove"))), - callee(functionDecl(hasName("sprintf"))), - callee(functionDecl(hasName("snprintf"))), - callee(functionDecl(hasName("scanf"))), - callee(functionDecl(hasName("sscanf"))), - callee(functionDecl(hasName("printf"))), - callee(functionDecl(hasName("printRow"))), - callee(functionDecl(hasName("puts"))), - callee(functionDecl(hasName("sleep"))), - callee(functionDecl(hasName("printResult"))), - callee(functionDecl(hasName("getchar"))), - callee(functionDecl(hasName("taos_print_row"))), - callee(functionDecl(hasName("fprintf"))), - callee(functionDecl(hasName("taosMemoryFreeClear"))), - callee(functionDecl(hasName("taosMemoryFree"))), - callee(functionDecl(hasName("taosArrayDestroy"))), - callee(functionDecl(hasName("taosArrayDestroyEx"))), - callee(functionDecl(hasName("TAOS_CHECK_RETURN"))) + #log and print functions + callee(functionDecl(matchesName(".*(log|Log|Print|Debug|Trace|Warn|Error|Info|Fatal).*"))), + # memory functions + callee(functionDecl(matchesName(".*memcpy.*"))), + callee(functionDecl(matchesName(".*strcpy.*"))), + callee(functionDecl(matchesName(".*strcat.*"))), + callee(functionDecl(matchesName(".*strncpy.*"))), + callee(functionDecl(matchesName(".*memset.*"))), + callee(functionDecl(matchesName(".*memmove.*"))), + # string functions + callee(functionDecl(matchesName(".*sprintf.*"))), + callee(functionDecl(matchesName(".*snprintf.*"))), + callee(functionDecl(matchesName(".*scanf.*"))), + callee(functionDecl(matchesName(".*sscanf.*"))), + callee(functionDecl(matchesName(".*printf.*"))), + callee(functionDecl(matchesName(".*printRow.*"))), + callee(functionDecl(matchesName(".*puts.*"))), + callee(functionDecl(matchesName(".*sleep.*"))), + callee(functionDecl(matchesName(".*printResult.*"))), + callee(functionDecl(matchesName(".*getchar.*"))), + callee(functionDecl(matchesName(".*taos_print_row.*"))), + callee(functionDecl(matchesName(".*fprintf.*"))), + callee(functionDecl(matchesName(".*taosMemoryFreeClear.*"))), + callee(functionDecl(matchesName(".*taosMemoryFree.*"))), + callee(functionDecl(matchesName(".*taosArrayDestroy.*"))), + callee(functionDecl(matchesName(".*taosArrayDestroyEx.*"))), + callee(functionDecl(matchesName(".*.*setThreadName.*"))), + callee(functionDecl(matchesName(".*pthread_setname_np.*"))) )) ) \ No newline at end of file diff --git a/test/ci/run.sh b/test/ci/run.sh index a4a94f4710b3..c2e8f0d6c554 100755 --- a/test/ci/run.sh +++ b/test/ci/run.sh @@ -457,17 +457,16 @@ function run_thread() { local build_dir=$log_dir/build_${hosts[index]} local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/" - mkdir "$build_dir" >/dev/null - if [ $? -eq 0 ]; then - if is_local_host "${hosts[index]}"; then - cmd="cp -rf ${remote_build_dir}/* ${build_dir}/" + + if is_local_host "${hosts[index]}"; then + mkdir "$build_dir" >/dev/null + cmd="cp -rf ${remote_build_dir}/* ${build_dir}/" + echo "$cmd" + bash -c "$cmd" >/dev/null 2>&1 || true + if [ -d "${remote_unit_test_log_dir}" ] && [ "$(ls -A "${remote_unit_test_log_dir}" 2>/dev/null)" ]; then + cmd="cp -rf ${remote_unit_test_log_dir}/* ${build_dir}/" echo "$cmd" bash -c "$cmd" >/dev/null 2>&1 || true - if [ -d "${remote_unit_test_log_dir}" ] && [ "$(ls -A "${remote_unit_test_log_dir}" 2>/dev/null)" ]; then - cmd="cp -rf ${remote_unit_test_log_dir}/* ${build_dir}/" - echo "$cmd" - bash -c "$cmd" >/dev/null 2>&1 || true - fi fi fi local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" diff --git a/test/ci/scan_file_path.py b/test/ci/scan_file_path.py index 371618295e38..0daa20076542 100644 --- a/test/ci/scan_file_path.py +++ b/test/ci/scan_file_path.py @@ -84,14 +84,14 @@ # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" -scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] +scan_dir_list = ["source", "include", "docs/examples"] # # all the c files path will be checked all_file_path = [] file_res_path = "" -SCAN_DIRS = ["source", "include", "docs/examples", "src/plugins"] +SCAN_DIRS = ["source", "include", "docs/examples"] SCAN_SKIP_FILE_LIST = [ "tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", @@ -248,21 +248,11 @@ def scan_one_file(file): if __name__ == "__main__": command_executor = CommandExecutor() - # get all the c files path - # scan_files_path("/root/TDinternal/community/source/") - # input_files(change_file_list) - # print(f"all_file_path:{all_file_path}") res = [] web_path = [] res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) - # create dir - # current_time = datetime.now().strftime("%Y%m%d%H%M%S") - # scan_result_path = os.path.join(scan_result_base_path, current_time) - # scan_result_path = scan_result_base_path - # if not os.path.exists(scan_result_path): - # os.makedirs(scan_result_path) - - # 优先用 -d 指定目录，否则用 -f 文件列表，否则默认目录 + + # use -d directory first, else -f file list, else default directories if change_file_list: input_files(change_file_list) elif scan_dir: @@ -275,43 +265,9 @@ def scan_one_file(file): scan_files_path(abs_dir) all_file_path = list(set(all_file_path)) - # 多进程并发扫描 with concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) as executor: results = list(executor.map(scan_one_file, all_file_path)) res.extend(results) - - # for file in all_file_path: - # cmd = f"clang-query-16 -p {compile_commands_path} {file} -f {clang_scan_rules_path} 2>&1 | grep -v 'error:' | grep -v 'warning:'" - # logger.debug(f"cmd:{cmd}") - # try: - # stdout, stderr = command_executor.execute(cmd) - # print(stderr) - # lines = stdout.split("\n") - # scan_valid = len(lines) >= 2 and (lines[-2].endswith("matches.") or lines[-2].endswith("match.")) - # if scan_valid: - # match_num = int(lines[-2].split(" ")[0]) - # logger.info("The match lines of file %s: %s" % (file, match_num)) - # this_file_res_path = save_scan_res(log_file_path, file, stdout, stderr) - # if match_num > 0: - # # logger.info(f"log_file_path: {log_file_path} ,file:{file}") - # index_tests = this_file_res_path.find("scan_log") - # if index_tests != -1: - # web_path_file = this_file_res_path[index_tests:] - # web_path_file = os.path.join(web_server, web_path_file) - # web_path.append(web_path_file) - # res.append([file, this_file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail']) - # else: - # logger.warning("The result of scan is invalid for: %s" % file) - # this_file_res_path = save_scan_res(log_file_path, file, stdout, stderr) - # res.append([file, this_file_res_path, 0, 'Invalid']) - # except Exception as e: - # logger.error("Execute command failed: %s" % e) - # this_file_res_path = "" - # res.append([file, this_file_res_path, 0, 'Error']) - # data = "" - # for item in res: - # data += item[0] + "," + str(item[1]) + "\n" - # logger.info("Csv data: %s" % data) write_csv(os.path.join(log_file_path, "scan_res.txt"), res) scan_result_log = f"{log_file_path}/scan_res.txt" # delete the first element of res @@ -330,7 +286,6 @@ def scan_one_file(file): logger.info("All files passed the scan.") if web_server: - # 打印 web_path for index, fail_item in enumerate(fail_files): file_res_path = fail_item[1] index_tests = file_res_path.find("scan_log") @@ -342,7 +297,6 @@ def scan_one_file(file): f"failed number: {index + 1}, failed_result_file: {web_path_file}" ) else: - # 打印本地路径 for index, fail_item in enumerate(fail_files): logger.error( f"failed number: {index + 1}, failed_result_file: {fail_item[1]}" diff --git a/tests/ci/dockerfile_ci b/tests/ci/dockerfile_ci index 158c5850a066..acdbef758cc6 100644 --- a/tests/ci/dockerfile_ci +++ b/tests/ci/dockerfile_ci @@ -1,44 +1,34 @@ -# Stage 1: Build Stage -FROM python:3.9 AS builder +FROM python:3.9.25 AS builder -# 设置工作目录 WORKDIR /home -# 定义变量 -ARG INSTALLERS_DIR=installers +RUN echo "deb http://deb.debian.org/debian bookworm main" > /etc/apt/sources.list \ + && echo "deb http://deb.debian.org/debian-security bookworm-security main" >> /etc/apt/sources.list \ + && echo "deb http://deb.debian.org/debian bookworm-updates main" >> /etc/apt/sources.list -# 复制必要的文件 COPY requirements.txt /home/ -COPY sources.list /etc/apt/ COPY id_ecdsa /root/.ssh/id_ecdsa COPY id_ecdsa.pub /root/.ssh/id_ecdsa.pub COPY id_ecdsa.pub /root/.ssh/authorized_keys COPY .gitconfig /root/.gitconfig -RUN rm -rf /etc/apt/sources.list.d/* - - -# 安装基础依赖（合并命令，减少镜像层） -RUN apt-key adv --keyserver hkp://pgpkeys.eu --recv-keys 3B4FE6ACC0B21F32 \ - && apt-key adv --keyserver hkp://pgpkeys.eu --recv-keys 871920D1991BC93C \ - && apt-get update \ +RUN apt-get update \ && apt-get install -y --no-install-recommends \ - locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev ruby-full \ - libgeos-dev libgoogle-glog-dev libsnappy-dev liblzma-dev libz-dev \ - libjansson-dev zlib1g pkg-config build-essential valgrind rsync vim \ + locales psmisc sudo tree libgeos-dev libgflags-dev ruby-full \ + libgoogle-glog-dev libsnappy-dev liblzma-dev libz-dev \ + libjansson-dev zlib1g pkg-config build-essential valgrind rsync vim \ libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg \ apt-transport-https ca-certificates software-properties-common iputils-ping \ - r-base r-base-dev clang-tools-16 wget lcov \ - && wget https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \ - && dpkg -i packages-microsoft-prod.deb \ - && rm packages-microsoft-prod.deb \ - && apt-get update \ - && apt-get install -y dotnet-sdk-6.0 \ + r-base r-base-dev clang-tools-16 wget lcov groff \ + && wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x dotnet-install.sh \ + && ./dotnet-install.sh --version 6.0.100 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ && sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen \ && locale-gen \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -# 配置 pip RUN pip3 config set global.index-url http://admin:123456@192.168.0.212:3141/admin/dev/+simple/ \ && pip3 config set global.trusted-host 192.168.0.212 \ && pip3 install --no-cache-dir -r /home/requirements.txt \ @@ -46,49 +36,52 @@ RUN pip3 config set global.index-url http://admin:123456@192.168.0.212:3141/admi taospy==2.8.8 taos-ws-py==0.6.5 pandas psutil codecov fabric2 requests faker simplejson toml \ pexpect tzlocal distro decorator loguru hyperloglog websockets -# 安装 Go、JDK、Maven、Node.js（优化解压路径） -ADD go1.23.4.linux-amd64.tar.gz /usr/local/ -ADD jdk-8u144-linux-x64.tar.gz /usr/local/ -ADD apache-maven-3.8.4-bin.tar.gz /usr/local/ -ADD node-v20.17.0-linux-x64.tar.xz /usr/local/ -ADD cmake-3.21.5-linux-x86_64.tar.gz /usr/local/ +ADD go1.23.4.linux-amd64.tar.gz \ + jdk-8u144-linux-x64.tar.gz \ + apache-maven-3.8.4-bin.tar.gz \ + node-v20.17.0-linux-x64.tar.xz \ + cmake-3.21.5-linux-x86_64.tar.gz \ + /usr/local/ + +# Use build-time ARGs and run-time ENVs for flexibility +ARG RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup +ARG RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static +ARG RUST_VERSION=1.90.0 -ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" -ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" -ENV RUST_VERSION=1.87.0 +ENV RUSTUP_UPDATE_ROOT=${RUSTUP_UPDATE_ROOT} +ENV RUSTUP_DIST_SERVER=${RUSTUP_DIST_SERVER} +ENV RUST_VERSION=${RUST_VERSION} -# 安装 Rust（优化安装过程） RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --verbose --profile minimal --component clippy,rustfmt --default-toolchain $RUST_VERSION COPY .cargo/config /root/.cargo/config -# 配置环境变量 ENV PATH="/usr/local/go/bin:/usr/local/node-v20.17.0-linux-x64/bin:/usr/local/apache-maven-3.8.4/bin:/usr/local/jdk1.8.0_144/bin:/root/.cargo/bin:/home/bin:/usr/local/cmake-3.21.5-linux-x86_64/bin:$PATH" ENV JAVA_HOME="/usr/local/jdk1.8.0_144" -ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" RUN go env -w GOPROXY=https://goproxy.cn ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 -# 配置 R 支持 Java 并安装 RJDBC RUN R CMD javareconf JAVA_HOME=${JAVA_HOME} JAVA=${JAVA_HOME}/bin/java JAVAC=${JAVA_HOME}/bin/javac JAVAH=${JAVA_HOME}/bin/javah JAR=${JAVA_HOME}/bin/jar \ && echo "install.packages(\"RJDBC\", repos=\"http://cran.us.r-project.org\")" | R --no-save -# 安装覆盖率工具 +# Install coverage tools RUN gem install coveralls-lcov \ && npm config -g set registry https://registry.npmmirror.com # COPY ../.nuget /root/.nuget # COPY ../.dotnet /root/.dotnet # COPY ../.npm /root/.npm -# 配置 SSH（优化配置并设置权限） +# Configure SSH (optimize configuration and set permissions) RUN mkdir -p /root/.ssh \ && chmod 700 /root/.ssh \ && echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \ && chmod 600 /root/.ssh/id_ecdsa \ - && chmod 600 /root/.ssh/authorized_keys + && chmod 600 /root/.ssh/authorized_keys \ + # for taosadapter start + && mkdir -p /etc/taos RUN sh -c "rm -f /etc/localtime;ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime;echo \"Asia/Shanghai\" >/etc/timezone" -# 复制应用代码 # COPY TDinternal /home/TDinternal COPY setup.sh /home/setup.sh -RUN chmod +x /home/setup.sh \ No newline at end of file +RUN chmod +x /home/setup.sh +COPY TDinternal /home/TDinternal \ No newline at end of file