diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..73e92d18d56 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,88 @@ +name: cbdb pipeline +on: + pull_request: + branches: + - main + +env: + BUILD_TYPE: "debug" + FTS_MODE: "internal_fts" + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + +permissions: + id-token: write + contents: read +jobs: + build: + runs-on: [self-hosted, example] + steps: + - uses: actions/checkout@v3 + - name: Run build script + run: | + echo $GITHUB_RUN_ID > BUILD_NUMBER + hd-ci/compile_cbdb.bash ${GEN_DOCKER_TAG} + - uses: actions/upload-artifact@v3 + with: + name: cbdb-variables + path: /opt/actions-runner/_work/cloudberrydb/cloudberrydb/cbdb-artifacts.txt + - uses: actions/upload-artifact@v3 + with: + name: cbdb-package + path: /tmp/server-build* + icw-test: + needs: build + runs-on: [self-hosted, example] + env: + MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world" + TEST_OS: "centos" + DUMP_DB: "true" + steps: + - uses: actions/checkout@v3 + with: + path: "gpdb_src" + - uses: actions/download-artifact@v3 + with: + name: cbdb-variables + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-package + path: /opt/ + - name: Run icw-test script + run: | + mkdir /code + cp -a gpdb_src/ /code + cd /code + echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER + gpdb_src/hd-ci/icw_cbdb.bash $FTS_MODE + icw-orca-test: + needs: build + runs-on: [self-hosted, example] + env: + MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world" + TEST_OS: "centos" + DUMP_DB: "true" + steps: + - uses: actions/checkout@v3 + with: + path: "gpdb_src" + - uses: actions/download-artifact@v3 + with: + name: cbdb-variables + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-package + path: /opt/ + - name: Run icw-orca-test script + run: | + mkdir /code + cp -a gpdb_src/ /code + cd /code + echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER + gpdb_src/hd-ci/icw_cbdb.bash $FTS_MODE + + + + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..e8e7c2c7457 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,137 @@ +name: cbdb release +on: + push: + tags: + - 'v*' + +env: + BUILD_TYPE: "release" + FTS_MODE: "internal_fts" + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + GEN_DOCKER_TAG: true + INSTALL_DIR: "/usr/local/cloudberry-db" + CLIENT_INSTALL_DIR: "/usr/local/cloudberry-clients" + + +jobs: + build: + runs-on: [self-hosted, default] + steps: + - uses: actions/checkout@v3 + - name: Run build script + run: | + echo $GITHUB_RUN_ID > BUILD_NUMBER + hd-ci/compile_cbdb.bash $GEN_DOCKER_TAG + - uses: actions/upload-artifact@v3 + with: + name: cbdb-variables + path: /opt/actions-runner/_work/cloudberrydb/cloudberrydb/cbdb-artifacts.txt + - uses: actions/upload-artifact@v3 + with: + name: cbdb-package + path: /tmp/server-build* + - uses: actions/upload-artifact@v3 + with: + name: cbdb-rpm + path: /tmp/cloudberry-db*.rpm + icw-test: + needs: build + runs-on: [self-hosted, example] + env: + MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world" + TEST_OS: "centos" + DUMP_DB: "true" + BUILD_TYPE: release + steps: + - uses: actions/checkout@v3 + with: + path: "gpdb_src" + - uses: actions/download-artifact@v3 + with: + name: cbdb-variables + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-package + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-rpm + path: /opt/ + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: us-east-1 + - name: Run icw-test script + run: | + mkdir /code + cp -a gpdb_src/ /code + cd /code + echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER + gpdb_src/hd-ci/icw_cbdb.bash internal_fts + icw-orca-test: + needs: build + runs-on: [self-hosted, example] + env: + MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world" + TEST_OS: "centos" + DUMP_DB: "true" + BUILD_TYPE: release + steps: + - uses: actions/checkout@v3 + with: + path: "gpdb_src" + - uses: actions/download-artifact@v3 + with: + name: cbdb-variables + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-package + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-rpm + path: /opt/ + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: us-east-1 + - name: Run icw-orca-test script + run: | + mkdir /code + cp -a gpdb_src/ /code + cd /code + echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER + gpdb_src/hd-ci/icw_cbdb.bash internal_fts + release: + name: Upload Release Asset + needs: ["icw-orca-test","icw-test"] + runs-on: ubuntu-latest + steps: + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false + - uses: actions/download-artifact@v3 + with: + name: cbdb-package + path: /opt/ + - uses: actions/download-artifact@v3 + with: + name: cbdb-rpm + path: /opt/ + - name: Upload Release Asset + id: upload-release-asset + uses: alexellis/upload-assets@master + env: + GITHUB_TOKEN: ${{ github.token }} + with: + asset_paths: '["/opt/server-build*","/opt/cloudberry-db-*"]' \ No newline at end of file diff --git a/concourse/scripts/common.bash b/concourse/scripts/common.bash index 987229613e4..7d0d023a604 100755 --- a/concourse/scripts/common.bash +++ b/concourse/scripts/common.bash @@ -40,15 +40,11 @@ function download_etcd() { etcd_file_name=etcd-${etcd_version}-linux-arm64 fi - etcd_download_url=https://artifactory.hashdata.xyz/artifactory/utility/${etcd_file_name}.tar.gz - - wget ${etcd_download_url} -O /tmp/${etcd_file_name}.tar.gz - tar -xvf /tmp/${etcd_file_name}.tar.gz -C /tmp + tar -xvf /opt/${etcd_file_name}.tar.gz -C /opt mkdir -p ${target_path} - \cp /tmp/${etcd_file_name}/etcd ${target_path} - \cp /tmp/${etcd_file_name}/etcdctl ${target_path} - rm -rf /tmp/${etcd_file_name} /tmp/${etcd_file_name}.tar.gz + \cp /opt/${etcd_file_name}/etcd ${target_path} + \cp /opt/${etcd_file_name}/etcdctl ${target_path} } function download_jansson() { @@ -59,16 +55,16 @@ function download_jansson() { fi jansson_file_name=jansson-${jansson_version} - wget https://artifactory.hashdata.xyz/artifactory/utility/${jansson_file_name}.tar.gz -O /tmp/${jansson_file_name}.tar.gz - tar -xvf /tmp/${jansson_file_name}.tar.gz -C /tmp + tar -xvf /opt/${jansson_file_name}.tar.gz -C /opt - pushd /tmp/${jansson_file_name} - ./configure --prefix=/tmp/jansson --disable-static + pushd /opt/${jansson_file_name} + ./configure --prefix=/opt/jansson --disable-static make && make install mkdir -p ${target_path} - \cp -p /tmp/jansson/lib/libjansson.so* ${target_path} - rm -rf /tmp/jansson /tmp/${jansson_file_name}.tar.gz /tmp/${jansson_file_name} + \cp -p /opt/jansson/lib/libjansson.so* ${target_path} + \cp -p /opt/jansson/include/jansson*.h /usr/include/ + rm -rf /opt/jansson popd } @@ -79,12 +75,10 @@ function download_java() { echo "invalid java target path!" && exit 1 fi java_file_name=jdk-${java_version}-linux-`arch` - wget http://artifactory.hashdata.xyz/artifactory/development/centos/7/`arch`/${java_file_name}.tar.gz -O /tmp/${java_file_name}.tar.gz mkdir -p ${target_path} - tar -xzf /tmp/${java_file_name}.tar.gz -C ${target_path} + tar -xzf /opt/${java_file_name}.tar.gz -C ${target_path} mv ${target_path}/jdk* ${target_path}/jdk - rm /tmp/${java_file_name}.tar.gz } ## ---------------------------------------------------------------------- @@ -93,7 +87,7 @@ function download_java() { function install_gpdb() { mkdir -p $INSTALL_DIR - tar -xzf bin_gpdb/bin_gpdb.tar.gz -C $INSTALL_DIR + tar -xzf /opt/bin_gpdb.tar.gz -C $INSTALL_DIR } function setup_configure_vars() { diff --git a/deploy/k8s/Dockerfile b/deploy/k8s/Dockerfile index 003e4cdae2b..d68b49017c0 100644 --- a/deploy/k8s/Dockerfile +++ b/deploy/k8s/Dockerfile @@ -1,31 +1,19 @@ -FROM docker.artifactory.hashdata.xyz/docker/cbdb:centosbase_k8s_1.0 +FROM cloudberrydb/cloudberrydb:centosbase_k8s_1.0 ARG PACKAGE ARG BUILD_TYPE LABEL description="dockerfile for cbdb k8s auto-deployment" -RUN curl -O http://artifactory.hashdata.xyz/artifactory/hashdata-repository/intermediate-artifacts/centos7/x86_64/cbdb/${BUILD_TYPE}/${PACKAGE} && \ - yum -y install R apr apr-devel apr-util automake autoconf bash bison bison-devel bzip2 bzip2-devel centos-release-scl curl flex flex-devel gcc gcc-c++ git gdb ibxml2 iproute krb5 krb5-devel less libcurl libcurl-devel libevent libevent-devel libxml2 libxml2-devel libyaml libzstd-devel libzstd.x86_64 make openldap openssh-client openssl openssl-devel openssl-libs perl python3-devel readline readline-devel rsync sed sudo tar vim wget which xerces-c-devel zip zip-devel zlib && \ +ADD ${PACKAGE} . +RUN yum -y install R apr apr-devel apr-util automake autoconf bash bison bison-devel bzip2 bzip2-devel centos-release-scl curl flex flex-devel gcc gcc-c++ git gdb ibxml2 iproute krb5 krb5-devel less libcurl libcurl-devel libevent libevent-devel libxml2 libxml2-devel libyaml libzstd-devel libzstd.x86_64 make openldap openssh-client openssl openssl-devel openssl-libs perl python3-devel readline readline-devel rsync sed sudo tar vim wget which xerces-c-devel zip zip-devel zlib && \ yum -y install epel-release.noarch && \ yum -y install libzstd.x86_64 && \ yum -y install devtoolset-9-gcc* && \ - wget https://artifactory.hashdata.xyz/artifactory/utility/etcd-v3.3.25-linux-amd64.tar.gz && \ + wget https://cbdb-deps.s3.amazonaws.com/etcd-v3.3.25-linux-amd64.tar.gz && \ tar zxvf etcd-v3.3.25-linux-amd64.tar.gz && \ mv etcd-v3.3.25-linux-amd64/etcd /usr/local/bin/etcd && \ mv etcd-v3.3.25-linux-amd64/etcdctl /usr/local/bin/etcdctl && \ rm -rf etcd-v3.3.25-linux-amd64.tar.gz && \ rpm -ivh ${PACKAGE} && \ rm ${PACKAGE} && \ - wget https://artifactory.hashdata.xyz/artifactory/utility/jansson-2.13.1.tar.gz && \ - tar -xvf jansson-2.13.1.tar.gz && \ - rm -rf jansson-2.13.1.tar.gz && \ - cd jansson-2.13.1 && \ - ./configure --prefix=/usr --disable-static && \ - make && \ - make install && \ - cp -f /usr/lib/libjansson.so* /usr/lib64/ && \ - wget -O /root/cmake-3.25.1-linux-x86_64.tar.gz https://artifactory.hashdata.xyz/artifactory/utility/cmake-3.25.1-linux-x86_64.tar.gz && \ - mkdir -p /root/cmake-3.25.1-linux-x86_64 && \ - tar -xvf /root/cmake-3.25.1-linux-x86_64.tar.gz -C /root/cmake-3.25.1-linux-x86_64 && \ - ln -s /root/cmake-3.25.1-linux-x86_64/cmake-3.25.1-linux-x86_64/bin/cmake /usr/bin/cmake && \ chmod u+s /bin/ping && \ mkdir -p /data0/data1/primary && \ mkdir -p /data0/data2/primary && \ diff --git a/gpMgmt/bin/pythonSrc/ext b/gpMgmt/bin/pythonSrc/ext deleted file mode 160000 index 135d649aef8..00000000000 --- a/gpMgmt/bin/pythonSrc/ext +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 135d649aef8b70eb4ee8ec824f869026ac3f2b28 diff --git a/gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz b/gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz new file mode 100644 index 00000000000..9c39e0acc34 Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/PyYAML-5.3.1.tar.gz b/gpMgmt/bin/pythonSrc/ext/PyYAML-5.3.1.tar.gz new file mode 100644 index 00000000000..915d67b38f2 Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/PyYAML-5.3.1.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz b/gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz new file mode 100644 index 00000000000..dc7a21a28f5 Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz b/gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz new file mode 100644 index 00000000000..4fdea77c71c Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz b/gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz new file mode 100644 index 00000000000..d24aa0d1afa Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz b/gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz new file mode 100644 index 00000000000..e8b1d804420 Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/pylint-0.21.0.tar.gz b/gpMgmt/bin/pythonSrc/ext/pylint-0.21.0.tar.gz new file mode 100644 index 00000000000..cf876d33316 Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/pylint-0.21.0.tar.gz differ diff --git a/gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz b/gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz new file mode 100644 index 00000000000..bdfc240ce1b Binary files /dev/null and b/gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz differ diff --git a/hd-ci/common.bash b/hd-ci/common.bash index e2aeff0a194..1243ef369a2 100644 --- a/hd-ci/common.bash +++ b/hd-ci/common.bash @@ -72,15 +72,9 @@ setup_cbdb() { download_cbdb_tar_package() { fts_mode=$1 - source "${SRC_PATH}"/cbdb-artifacts.txt + source /opt/cbdb-artifacts.txt mkdir -p "${ROOT_PATH}"/bin_gpdb - local download_url=${fts_mode:+$(if [ "${fts_mode}" == "external_fts" ]; then echo "${fts_mode}_"; fi)}TARGZ_cbdb_${OS_TYPE}_${OS_ARCH}_url - - - curl -sSfL \ - -o "${ROOT_PATH}/bin_gpdb/bin_gpdb.tar.gz" \ - -z "${ROOT_PATH}/bin_gpdb/bin_gpdb.tar.gz" \ - "${!download_url}" + mv /opt/${internal_tar_download_url##*/} "/opt/bin_gpdb.tar.gz" } setup_env() { diff --git a/hd-ci/compile_cbdb.bash b/hd-ci/compile_cbdb.bash index b9cbc9a9db1..581b909156b 100755 --- a/hd-ci/compile_cbdb.bash +++ b/hd-ci/compile_cbdb.bash @@ -2,6 +2,7 @@ set -exo pipefail BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source /etc/profile source "${BASE_DIR}"/common.bash CBDB_VERSION=$(${SRC_PATH}/getversion --short) @@ -31,32 +32,44 @@ function compile_cbdb() { function upload_cbdb_tar_and_rpm_package() { fts_mode=$1 generate_docker_tag=$2 - tar_package_path=$(ls "${ROOT_PATH}"/gpdb_artifacts/server-build*) - tar_upload_file_name=$(basename "${tar_package_path}" ".tar.gz")-"${BUILD_NUMBER}"-"${BUILD_TYPE}"${fts_mode:+$(if [ "${fts_mode}" = "external_fts" ]; then echo "-${fts_mode}"; fi)}.tar.gz - tar_download_url="$BUCKET_INTERMEDIATE/${OS_TYPE}/${OS_ARCH}/cbdb/${BUILD_TYPE}/${tar_upload_file_name}" - tar_download_url_latest="$BUCKET_INTERMEDIATE/${OS_TYPE}/${OS_ARCH}/cbdb/${BUILD_TYPE}/bin_cbdb_latest${fts_mode:+$(if [ "${fts_mode}" = "external_fts" ]; then echo "_${fts_mode}"; fi)}.tar.gz" + tar_package_path=$(ls "${ROOT_PATH}"/cloudberrydb/gpdb_artifacts/server-build*) + cp $tar_package_path /tmp + + rpm_package_path=$(ls ~/rpmbuild/RPMS/"${OS_ARCH}"/cloudberry-db*.rpm) + + cp $rpm_package_path /tmp + if [[ ${fts_mode} == "internal_fts" ]]; then + cat <>"${SRC_PATH}/cbdb-artifacts.txt" +os_type=${OS_TYPE_EXT} +os_arch=${OS_ARCH} +cbdb_major_version=1.x +cbdb_version=${CBDB_VERSION} +build_number=${BUILD_NUMBER} +build_type=${BUILD_TYPE} +internal_tar_download_url=${tar_package_path} +internal_rpm_download_url=${rpm_package_path} +CBDB_VERSION=${CBDB_VERSION} +EOF + else + cat <>"${SRC_PATH}/cbdb-artifacts.txt" +external_fts_RELEASE_cbdb_${OS_TYPE}_${OS_ARCH}_url=${rpm_download_url} +external_fts_TARGZ_cbdb_${OS_TYPE}_${OS_ARCH}_url=${tar_download_url} +EOF + fi - # upload tar.gz file - curl -s -S -f -u "${ARTIFACTORY_USERNAME}":"${ARTIFACTORY_PASSWORD}" \ - -X PUT "${tar_download_url}" \ - -T "${tar_package_path}" - curl -s -S -f -u "${ARTIFACTORY_USERNAME}":"${ARTIFACTORY_PASSWORD}" \ - -X PUT "${tar_download_url_latest}" \ - -T "${tar_package_path}" + cat "${SRC_PATH}/cbdb-artifacts.txt" +} + +function upload_external_fts_cbdb_tar_and_rpm_package() { + fts_mode=$1 + generate_docker_tag=$2 + tar_package_path=$(ls "${ROOT_PATH}"/cloudberrydb/gpdb_artifacts/server-build*) + tar_upload_file_name=$(basename "${tar_package_path}" ".tar.gz")-"${BUILD_NUMBER}"-"${BUILD_TYPE}"${fts_mode:+$(if [ "${fts_mode}" = "external_fts" ]; then echo "-${fts_mode}"; fi)}.tar.gz + cp $tar_package_path /tmp/$tar_upload_file_name rpm_package_path=$(ls ~/rpmbuild/RPMS/"${OS_ARCH}"/cloudberry-db*.rpm) rpm_upload_file_name=$(basename "${rpm_package_path}" ".rpm")-"${BUILD_NUMBER}"-"${BUILD_TYPE}"${fts_mode:+$(if [ "${fts_mode}" = "external_fts" ]; then echo "-${fts_mode}"; fi)}.rpm - rpm_download_url="$BUCKET_INTERMEDIATE/${OS_TYPE}/${OS_ARCH}/cbdb/${BUILD_TYPE}/${rpm_upload_file_name}" - rpm_download_url_latest="$BUCKET_INTERMEDIATE/${OS_TYPE}/${OS_ARCH}/cbdb/${BUILD_TYPE}/cbdb_latest${fts_mode:+$(if [ "${fts_mode}" = "external_fts" ]; then echo "_${fts_mode}"; fi)}.rpm" - - # upload rpm file - curl -s -S -f -u "${ARTIFACTORY_USERNAME}":"${ARTIFACTORY_PASSWORD}" \ - -X PUT "${rpm_download_url}" \ - -T "${rpm_package_path}" - curl -s -S -f -u "${ARTIFACTORY_USERNAME}":"${ARTIFACTORY_PASSWORD}" \ - -X PUT "${rpm_download_url_latest}" \ - -T "${rpm_package_path}" - + cp $rpm_package_path /tmp/$rpm_upload_file_name if [[ ${fts_mode} == "internal_fts" ]]; then cat <>"${SRC_PATH}/cbdb-artifacts.txt" os_type=${OS_TYPE_EXT} @@ -65,12 +78,8 @@ cbdb_major_version=1.x cbdb_version=${CBDB_VERSION} build_number=${BUILD_NUMBER} build_type=${BUILD_TYPE} -internal_tar_download_url=${tar_download_url} -internal_rpm_download_url=${rpm_download_url} -public_rpm_download_url=${rpm_download_url} -RELEASE_cbdb_${OS_TYPE}_${OS_ARCH}_name=${rpm_package_path##*/} -RELEASE_cbdb_${OS_TYPE}_${OS_ARCH}_url=${rpm_download_url} -TARGZ_cbdb_${OS_TYPE}_${OS_ARCH}_url=${tar_download_url} +internal_tar_download_url=${tar_package_path} +internal_rpm_download_url=${rpm_package_path} CBDB_VERSION=${CBDB_VERSION} EOF else @@ -82,14 +91,17 @@ EOF if [ "${generate_docker_tag}" = "true" ]; then release_image_k8s ${rpm_package_path} ${rpm_upload_file_name} + echo ${rpm_package_path} ${rpm_upload_file_name} fi cat "${SRC_PATH}/cbdb-artifacts.txt" } + + function package_create_rpm() { mkdir -p ~/rpmbuild/SOURCES/ - cp -a "${ROOT_PATH}/gpdb_artifacts/bin_gpdb.tar.gz" ~/rpmbuild/SOURCES/bin_cbdb.tar.gz + cp -a "${ROOT_PATH}/cloudberrydb/gpdb_artifacts/bin_gpdb.tar.gz" ~/rpmbuild/SOURCES/bin_cbdb.tar.gz version=$(echo "${CBDB_VERSION}" | tr '-' '_') PATH=/usr/bin:$PATH rpmbuild \ @@ -100,17 +112,16 @@ function package_create_rpm() { function release_image_k8s() { rpm_package_path=$1 - rpm_upload_file_name=$2 pushd ${SRC_PATH}/deploy/k8s - echo "{ \"insecure-registries\": [\"${DOCKER_REPO}\"] }" >> /etc/docker/daemon.json - echo "push new docker image to ${DOCKER_REPO}" + cp /root/rpmbuild/RPMS/x86_64/cloudberry-db-*.rpm . + rpm_package_name=`ls cloudberry-db-*.rpm` systemctl daemon-reload systemctl restart docker tag_name=$(basename "${rpm_package_path}" ".rpm")-"${BUILD_NUMBER}"-"${BUILD_TYPE}" tag_name=$(echo "${tag_name}" | sed 's/\+/-/g') - docker build . -t ${DOCKER_REPO}/docker/cbdb:${tag_name} --build-arg PACKAGE=${rpm_upload_file_name} --build-arg BUILD_TYPE=${BUILD_TYPE} - echo ${ARTIFACTORY_PASSWORD} | docker login --username ${ARTIFACTORY_USERNAME} --password-stdin ${DOCKER_REPO} - docker push ${DOCKER_REPO}/docker/cbdb:${tag_name} + docker build . -t cloudberrydb/cloudberrydb:${tag_name} --build-arg PACKAGE=${rpm_package_name} --build-arg BUILD_TYPE=${BUILD_TYPE} + echo ${DOCKERHUB_PASSWORD} | docker login --username ${DOCKERHUB_USERNAME} --password-stdin + docker push cloudberrydb/cloudberrydb:${tag_name} popd } @@ -124,7 +135,7 @@ function release_rpm_external_fts() { gen_docker_tag=$1 compile_cbdb "external_fts" package_create_rpm - upload_cbdb_tar_and_rpm_package "external_fts" ${gen_docker_tag} + upload_external_fts_cbdb_tar_and_rpm_package "external_fts" ${gen_docker_tag} } main() { @@ -135,16 +146,16 @@ main() { cp -r ${ROOT_PATH}/${gpdb_dir_name} ${ROOT_PATH}/gpdb_src_tmp popd - release_rpm_internal_fts - + release_rpm_internal_fts pushd ${ROOT_PATH} cp ${gpdb_dir_name}/cbdb-artifacts.txt . rm -rf ${gpdb_dir_name} ~/rpmbuild ${OUTPUT_ARTIFACT_DIR:=gpdb_artifacts} mv gpdb_src_tmp ${gpdb_dir_name} mv cbdb-artifacts.txt ${gpdb_dir_name} popd - release_rpm_external_fts $1 + + } main $1 diff --git a/hd-ci/icw_cbdb.bash b/hd-ci/icw_cbdb.bash index 1ca7c84174b..3fed90fea84 100755 --- a/hd-ci/icw_cbdb.bash +++ b/hd-ci/icw_cbdb.bash @@ -7,7 +7,8 @@ source "${BASE_DIR}"/common.bash mkdir -p "${ROOT_PATH}/sqldump" compile_jansson() { - wget https://artifactory.hashdata.xyz/artifactory/utility/jansson-2.13.1.tar.gz && \ + # wget https://artifactory.hashdata.xyz/artifactory/utility/jansson-2.13.1.tar.gz && \ + cp /opt/jansson-2.13.1.tar.gz . && \ tar -xvf jansson-2.13.1.tar.gz && \ rm -rf jansson-2.13.1.tar.gz && \ pushd . @@ -27,12 +28,10 @@ function download_etcd() { else ETCD_FILE_NAME=etcd-v3.3.25-linux-arm64 fi - ETCD_DOWNLOAD_URL=https://artifactory.hashdata.xyz/artifactory/utility/${ETCD_FILE_NAME}.tar.gz - wget ${ETCD_DOWNLOAD_URL} -O /tmp/${ETCD_FILE_NAME}.tar.gz - tar -xvf /tmp/${ETCD_FILE_NAME}.tar.gz -C /tmp - cp /tmp/${ETCD_FILE_NAME}/etcd ${GREENPLUM_INSTALL_DIR}/bin - cp /tmp/${ETCD_FILE_NAME}/etcdctl ${GREENPLUM_INSTALL_DIR}/bin - rm -rf /tmp/${ETCD_FILE_NAME} /tmp/${ETCD_FILE_NAME}.tar.gz + + tar -xvf /opt/${ETCD_FILE_NAME}.tar.gz -C /opt + cp /opt/${ETCD_FILE_NAME}/etcd ${GREENPLUM_INSTALL_DIR}/bin + cp /opt/${ETCD_FILE_NAME}/etcdctl ${GREENPLUM_INSTALL_DIR}/bin export ETCD_UNSUPPORTED_ARCH=arm64 } @@ -60,10 +59,7 @@ prepare_release() { pushd "${ROOT_PATH}" mkdir -p bin_gpdb_rpm && cd bin_gpdb_rpm - local rpm_download_url=${fts_mode:+$(if [ "${fts_mode}" == "external_fts" ]; then echo "${fts_mode}_"; fi)}RELEASE_cbdb_${OS_TYPE}_${OS_ARCH}_url - [ -z "$rpm_download_url" ] && echo "rpm download url is not set" >&2 && exit 1 - curl -O "${!rpm_download_url}" - yum install -y c*.rpm + yum install -y /opt/c*.rpm cd "$INSTALL_DIR" tar czf "${ROOT_PATH}/bin_gpdb_rpm/bin_gpdb.tar.gz" ./* popd diff --git a/src/backend/cdb/test/Makefile b/src/backend/cdb/test/Makefile index 4e43c275409..6a33e5d8066 100644 --- a/src/backend/cdb/test/Makefile +++ b/src/backend/cdb/test/Makefile @@ -6,8 +6,6 @@ TARGETS= cdbdistributedsnapshot TARGETS += cdbappendonlyxlog -TARGETS += cdbutils - # There are some functions both defined in src/backend/storage/file/fd.c # and libpgcommon.a which contains object files with FRONTEND defined, # for use by client applications. @@ -26,11 +24,3 @@ cdbappendonlyxlog.t: \ $(MOCK_DIR)/backend/access/transam/xlogutils_mock.o \ $(MOCK_DIR)/backend/access/hash/hash_mock.o \ $(MOCK_DIR)/backend/utils/fmgr/fmgr_mock.o - -cdbutils.t: \ - $(MOCK_DIR)/backend/storage/lmgr/lwlock_mock.o \ - $(MOCK_DIR)/backend/catalog/namespace_mock.o \ - $(MOCK_DIR)/backend/access/table/table_mock.o \ - $(MOCK_DIR)/backend/access/hash/hash_mock.o \ - $(MOCK_DIR)/backend/utils/fmgr/fmgr_mock.o \ - $(MOCK_DIR)/backend/storage/ipc/shmem_mock.o \ No newline at end of file diff --git a/src/backend/cdb/test/cdbutils_test.c b/src/backend/cdb/test/cdbutils_test.c deleted file mode 100644 index 693d0dc8167..00000000000 --- a/src/backend/cdb/test/cdbutils_test.c +++ /dev/null @@ -1,609 +0,0 @@ -#include -#include -#include -#include "cmockery.h" -#include "cdb/cdbutil.h" -#include "utils/etcd.h" -#include "utils/memutils.h" -#include "storage/lockdefs.h" -#include "storage/lwlock.h" -#include "common/etcdutils.h" - -#ifndef USE_INTERNAL_FTS - -static MemoryContext testMemoryContext = NULL; - -static -void expect_heap_open_and_close(void) { - expect_string_count(RelnameGetRelid, relname, GpSegmentConfigRelationName, -1); - will_return_count(RelnameGetRelid, 1, -1); - - expect_value_count(table_open, relationId, 1, -1); - expect_value_count(table_open, lockmode, AccessShareLock, -1); - will_return_count(table_open, NULL, -1); - - expect_any_count(table_close, relation, -1); - expect_value_count(table_close, lockmode, NoLock, -1); - will_be_called_count(table_close, -1); -} - -static -void expect_lock_acquire_and_release(void) { - expect_value_count(LWLockAcquire, lock, CdbConfigCacheLock, -1); - expect_value_count(LWLockAcquire, mode, LW_EXCLUSIVE, -1); - will_return_count(LWLockAcquire, true, -1); - - expect_value_count(LWLockRelease, lock, CdbConfigCacheLock, -1); - will_be_called_count(LWLockRelease, -1); -} - -static void enable_segment_cache() { - gp_etcd_enable_cache = true; -} - -static void disable_segment_cache() { - gp_etcd_enable_cache = false; -} - -static void clean_segments_with_cache() { - enable_segment_cache(); - cleanSegments(); - disable_segment_cache(); -} - -static void -SetupDataStructures(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - - if (NULL == TopMemoryContext) - { - assert_true(NULL == testMemoryContext); - MemoryContextInit(); - - testMemoryContext = AllocSetContextCreate(TopMemoryContext, - "Test Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - - MemoryContextSwitchTo(testMemoryContext); - - TopMemoryContext = testMemoryContext; - } - - // Init global etcd config - char *endpoints = GP_ETCD_TEST_SINGLE_ENDPOINTS; - gp_etcd_endpoints = strdup(endpoints); - gp_etcd_account_id = GP_ETCD_ACCOUNT_ID_DEFAULT; - gp_etcd_cluster_id = GP_ETCD_CLUSTER_ID_DEFAULT; - gp_etcd_namespace = GP_ETCD_NAMESPACE_DEFAULT; - assert_true(NULL != testMemoryContext && - CurrentMemoryContext == testMemoryContext); - - int total_dbs = 0; - expect_lock_acquire_and_release(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - - if (total_dbs != 0) { - expect_heap_open_and_close(); - clean_segments_with_cache(); - } - - if (config_cached) { - pfree(config_cached); - config_cached = NULL; - } -} - -/* - * cleans up memory by reseting testMemoryContext - */ -static void -TeardownDataStructures(void **state) -{ - expect_lock_acquire_and_release(); - expect_heap_open_and_close(); - clean_segments_with_cache(); - if (gp_etcd_endpoints) { - free(gp_etcd_endpoints); - gp_etcd_endpoints = NULL; - } -} - -void * cache_mem = NULL; - -static void -test_cdb_utils_init_cache(void **state) -{ - /* init cache */ - cache_mem = malloc(ShmemSegmentConfigsCacheSize()); - memset(cache_mem, 0, ShmemSegmentConfigsCacheSize()); - - expect_value(ShmemAlloc, size, ShmemSegmentConfigsCacheSize()); - will_return(ShmemAlloc, cache_mem); - - ShmemSegmentConfigsCacheAllocation(); - enable_segment_cache(); - assert_true(isSegmentConfigsCacheEnable()); - - /* disable cache and check */ - disable_segment_cache(); - assert_false(isSegmentConfigsCacheEnable()); - assert_false(isSegmentConfigsCached()); -} - -static void -test_cdb_utils_segment_ops(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - int total_dbs = 0; - - disable_segment_cache(); - expect_heap_open_and_close(); - expect_lock_acquire_and_release(); - - /* make sure env is clean */ - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* add segment-1 */ - GpSegConfigEntry config_entry; - config_entry.dbid = 1; - config_entry.segindex = 0; - config_entry.role = GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY; - config_entry.preferred_role = GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY; - config_entry.status = GP_SEGMENT_CONFIGURATION_STATUS_UP; - config_entry.mode = GP_SEGMENT_CONFIGURATION_MODE_INSYNC; - config_entry.port = 10000; - config_entry.address = "localhost"; - config_entry.hostname = "localhost"; - config_entry.datadir = "null"; - - addSegment(&config_entry); - - /* read segment configuration cache and check */ - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_int_equal(config_cached[0].dbid, config_entry.dbid); - assert_string_equal(config_cached[0].datadir, config_entry.datadir); - pfree(config_cached); - - /* rewrite segment-1 and segment-2 */ - rewriteSegments("1 0 p p n d 10000 localhost localhost datadir1\n"\ - "2 0 m m n u 10001 localhost localhost datadir2\n", true); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_int_equal(config_cached[0].dbid, 1); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 2); - assert_string_equal(config_cached[1].datadir, "datadir2"); - pfree(config_cached); - - /* delete segment-2 */ - delSegment(2); - - /* read segment configuration cache and check */ - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_int_equal(config_cached[0].dbid, 1); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_DOWN); - pfree(config_cached); - - /* update segment mode and check cache have been updated */ - updateSegmentModeStatus(1, GP_SEGMENT_CONFIGURATION_MODE_INSYNC, GP_SEGMENT_CONFIGURATION_STATUS_UP); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_int_equal(config_cached[0].dbid, 1); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - pfree(config_cached); - - /* rewrite segment-51 segment-52 and make sure cache have been updated */ - rewriteSegments("51 -1 p p n u 10000 localhost localhost datadir1\n" \ - "52 -1 m m n u 10001 localhost localhost datadir2\n", true); - activateStandby(52, 51); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 52); - assert_true(config_cached[0].role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].preferred_role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_string_equal(config_cached[0].datadir, "datadir2"); - pfree(config_cached); - - /* clean up and check */ - clean_segments_with_cache(); - assert_false(isSegmentConfigsCached()); -} - -static void -test_cdb_utils_segment_cache_add_segment(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - char * cached_buff = NULL; - int total_dbs = 0; - - enable_segment_cache(); - expect_heap_open_and_close(); - expect_lock_acquire_and_release(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* add segment-1 */ - GpSegConfigEntry config_entry; - config_entry.dbid = 1; - config_entry.segindex = 0; - config_entry.role = GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY; - config_entry.preferred_role = GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY; - config_entry.status = GP_SEGMENT_CONFIGURATION_STATUS_UP; - config_entry.mode = GP_SEGMENT_CONFIGURATION_MODE_INSYNC; - config_entry.port = 10000; - config_entry.address = "localhost"; - config_entry.hostname = "localhost"; - config_entry.datadir = "null"; - - addSegment(&config_entry); - - /* read segment configuration cache and check */ - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached->dbid, config_entry.dbid); - assert_string_equal(config_cached->datadir, config_entry.datadir); - pfree(config_cached); - pfree(cached_buff); - - /* disable read configuration from cache and check */ - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(config_cached->dbid, config_entry.dbid); - assert_string_equal(config_cached->datadir, config_entry.datadir); - pfree(config_cached); - enable_segment_cache(); - - - /* add segment-100 */ - config_entry.dbid = 100; - config_entry.datadir = "datadir-100"; - addSegment(&config_entry); - - /* read segment configuration cache and check */ - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 1); - assert_string_equal(config_cached[0].datadir, "null"); - assert_int_equal(config_cached[1].dbid, config_entry.dbid); - assert_string_equal(config_cached[1].datadir, config_entry.datadir); - pfree(config_cached); - pfree(cached_buff); - - /* disable read from cache and check */ - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_int_equal(config_cached[0].dbid, 1); - assert_string_equal(config_cached[0].datadir, "null"); - assert_int_equal(config_cached[1].dbid, config_entry.dbid); - assert_string_equal(config_cached[1].datadir, config_entry.datadir); - pfree(config_cached); - - /* clean up and check */ - clean_segments_with_cache(); - assert_false(isSegmentConfigsCached()); -} - -static void -test_cdb_utils_segment_cache_del_segment(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - char * cached_buff = NULL; - int total_dbs = 0; - - disable_segment_cache(); - expect_lock_acquire_and_release(); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - - enable_segment_cache(); - - expect_heap_open_and_close(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* rewrite segment-11 segment-21 segment-31 */ - rewriteSegments("11 0 p p n u 10000 localhost localhost datadir1\n" \ - "21 0 m m n u 10001 localhost localhost datadir2\n" \ - "31 0 m m n u 10001 localhost localhost datadir3\n", true); - - /* read from etcd cache and check */ - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 3); - assert_true(cached_buff != NULL); - - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 3); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 11); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 21); - assert_string_equal(config_cached[1].datadir, "datadir2"); - assert_int_equal(config_cached[2].dbid, 31); - assert_string_equal(config_cached[2].datadir, "datadir3"); - pfree(cached_buff); - pfree(config_cached); - - /* delete segment-21 */ - delSegment(21); - - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 11); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 31); - assert_string_equal(config_cached[1].datadir, "datadir3"); - pfree(cached_buff); - pfree(config_cached); - - /* disable read from cache and check */ - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 11); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 31); - assert_string_equal(config_cached[1].datadir, "datadir3"); - pfree(config_cached); - enable_segment_cache(); - - /* delete segment-11 */ - delSegment(11); - - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 31); - assert_string_equal(config_cached[0].datadir, "datadir3"); - pfree(cached_buff); - pfree(config_cached); - - /* disable read from cache and check */ - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 31); - assert_string_equal(config_cached[0].datadir, "datadir3"); - pfree(config_cached); - enable_segment_cache(); - - /* delete segment-31 */ - delSegment(31); - - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(cached_buff == NULL); - - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* no need clean, cause nothing in ETCD */ - assert_false(isSegmentConfigsCached()); -} - -static void -test_cdb_utils_segment_cache_activate_standby(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - char * cached_buff = NULL; - int total_dbs = 0; - - enable_segment_cache(); - expect_heap_open_and_close(); - expect_lock_acquire_and_release(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* rewrite master-51 and standby-52 */ - rewriteSegments("51 -1 p p n u 10000 localhost localhost datadir1\n" \ - "52 -1 m m n u 10001 localhost localhost datadir2\n", true); - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(cached_buff != NULL); - pfree(cached_buff); - - /* do active standby and check */ - activateStandby(52, 51); - - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 52); - assert_true(config_cached[0].role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].preferred_role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_string_equal(config_cached[0].datadir, "datadir2"); - pfree(cached_buff); - pfree(config_cached); - - /* disable read from cache and check */ - disable_segment_cache(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 1); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 52); - assert_true(config_cached[0].role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].preferred_role == GP_SEGMENT_CONFIGURATION_ROLE_PRIMARY); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_string_equal(config_cached[0].datadir, "datadir2"); - pfree(config_cached); - enable_segment_cache(); - - /* clean up and check */ - clean_segments_with_cache(); - assert_false(isSegmentConfigsCached()); - -} - -static void -test_cdb_utils_segment_cache_update_segment_mode_status(void **state) -{ - GpSegConfigEntry *config_cached = NULL; - char * cached_buff = NULL; - int total_dbs = 0; - - enable_segment_cache(); - expect_heap_open_and_close(); - expect_lock_acquire_and_release(); - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 0); - assert_true(config_cached == NULL); - - /* rewrite segment-1 and segment-2 */ - rewriteSegments("51 4 p p n u 10000 localhost localhost datadir1\n" \ - "52 4 m m s u 10001 localhost localhost datadir2\n", true); - - /* update segment mode and check cache have been updated */ - updateSegmentModeStatus(51, GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC, GP_SEGMENT_CONFIGURATION_STATUS_DOWN); - - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 51); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_DOWN); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 52); - assert_true(config_cached[1].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[1].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[1].datadir, "datadir2"); - pfree(cached_buff); - pfree(config_cached); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 51); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_DOWN); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 52); - assert_true(config_cached[1].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[1].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[1].datadir, "datadir2"); - pfree(config_cached); - - updateSegmentModeStatus(51, GP_SEGMENT_CONFIGURATION_MODE_INSYNC, GP_SEGMENT_CONFIGURATION_STATUS_UP); - cached_buff = readSegmentConfigsCache(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(cached_buff != NULL); - config_cached = readGpSegConfig(cached_buff, &total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 51); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 52); - assert_true(config_cached[1].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[1].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[1].datadir, "datadir2"); - pfree(cached_buff); - pfree(config_cached); - - config_cached = readGpSegConfigFromETCDAllowNull(&total_dbs); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(total_dbs, 2); - assert_true(config_cached != NULL); - assert_int_equal(config_cached[0].dbid, 51); - assert_true(config_cached[0].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[0].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[0].datadir, "datadir1"); - assert_int_equal(config_cached[1].dbid, 52); - assert_true(config_cached[1].mode == GP_SEGMENT_CONFIGURATION_MODE_INSYNC); - assert_true(config_cached[1].status == GP_SEGMENT_CONFIGURATION_STATUS_UP); - assert_string_equal(config_cached[1].datadir, "datadir2"); - pfree(config_cached); - - clean_segments_with_cache(); - assert_false(isSegmentConfigsCached()); -} - - -int -main(int argc, char* argv[]) -{ - int rc; - cmockery_parse_arguments(argc, argv); - - disable_segment_cache(); - const UnitTest tests[] = { - /* test_cdb_utils_init_cache must be first test case - * other tests will used segment configuration cache which need init in test_cdb_utils_init_cache - */ - unit_test_setup_teardown(test_cdb_utils_init_cache, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_cdb_utils_segment_ops, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_cdb_utils_segment_cache_add_segment, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_cdb_utils_segment_cache_del_segment, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_cdb_utils_segment_cache_activate_standby, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_cdb_utils_segment_cache_update_segment_mode_status, SetupDataStructures, TeardownDataStructures) - }; - rc = run_tests(tests); - - /* clean memory at last */ - MemoryContextReset(testMemoryContext); - testMemoryContext = NULL; - TopMemoryContext = NULL; - - /* release cache buffer */ - if (cache_mem) - free(cache_mem); - - return rc; -} -#else - -int -main(int argc, char* argv[]) -{ - return 0; -} -#endif \ No newline at end of file diff --git a/src/backend/utils/etcd_lib/test/Makefile b/src/backend/utils/etcd_lib/test/Makefile deleted file mode 100755 index 6b8b0c6a827..00000000000 --- a/src/backend/utils/etcd_lib/test/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -subdir=src/backend/utils/etcd_lib -top_builddir=../../../../.. -include $(top_builddir)/src/Makefile.global - -TARGETS=etcdlib - -include $(top_srcdir)/src/backend/mock.mk -LDFLAGS := -luuid $(LDFLAGS) -CFLAGS := -o0 $(CFLAGS) - -etcdlib.t: $(MOCK_DIR)/backend/utils/error/elog_mock.o \ - $(MOCK_DIR)/backend/access/hash/hashpage_mock.o \ - $(MOCK_DIR)/backend/utils/fmgr/fmgr_mock.o \ - $(MOCK_DIR)/backend/storage/ipc/shmem_mock.o \ - $(MOCK_DIR)/backend/storage/lmgr/lwlock_mock.o diff --git a/src/backend/utils/etcd_lib/test/etcdlib_test.c b/src/backend/utils/etcd_lib/test/etcdlib_test.c deleted file mode 100644 index 43e2d2d42d5..00000000000 --- a/src/backend/utils/etcd_lib/test/etcdlib_test.c +++ /dev/null @@ -1,673 +0,0 @@ -/** - * Test program for testing the etcdlib. - * Prerequisite is that etcdlib is started on localhost on default port (2379) - * tested with etcd 2.3.7 - */ - - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "postgres.h" -#include "cmockery.h" -#include "utils/etcd.h" -#include "lib/stringinfo.h" -#include "nodes/nodes.h" -#include "utils/memutils.h" -#include "common/etcdutils.h" - -#ifndef USE_INTERNAL_FTS - -static etcdlib_t *etcdlib; -static MemoryContext testMemoryContext = NULL; -static pthread_mutex_t lease_lock = PTHREAD_MUTEX_INITIALIZER; - -#define SIMPLEKEY "simplekey" -#define SIMPLEVALUE "testvalue" -#define DIRECTORYKEY "/cbdb/fts/default/e3cb5400-9589-918d-c178-82d500deac6e/7bc05356-67f9-49fe-804e-12fe30b093ef/fts_dump_file_key" -#define SIMPLELOCK "simplelock" -#define LEASETIMEOUT 12 -#define ETCD_TEST_HOST "192.168.180.86" -#define ETCD_TEST_PORT 2379 -#define ETCD_TEST_SINGLE_ENDPOINTS_NUM 1 -#define ETCD_TEST_MULTI_ENDPOINTS_NUM 3 -#define UUID_LEN 37 -#define ETCD_LOCK_LEN 256 -#define ETCD_LOCK_KEY "fts_ha_lock" -#define METADATA_DIR_PREFIX "/cbdb/fts" -#define NAMESPACE "default" - -/* - * * Mock PG_RE_THROW, because we are not using real elog.o. - * * The closest mockery is to call siglongjmp(). - * */ -#undef PG_RE_THROW -#define PG_RE_THROW() siglongjmp(*PG_exception_stack, 1) - -static char test_endpoint_list[ETCD_TEST_MULTI_ENDPOINTS_NUM][GP_ETCD_HOSTNAME_LEN] = {"192.168.180.86", "192.168.180.87", "192.168.180.88"}; -static char test_failover_endpoint_list[ETCD_TEST_MULTI_ENDPOINTS_NUM+1][GP_ETCD_HOSTNAME_LEN] = {"127.0.0.1", "192.168.180.86", "192.168.180.87", "192.168.180.88"}; - -static void -_errfinish_impl() -{ - PG_RE_THROW(); -} - -#define EXPECT_EREPORT(LOG_LEVEL) \ - if (LOG_LEVEL < ERROR )\ - { \ - expect_value(errstart, elevel, (LOG_LEVEL)); \ - expect_any(errstart, domain); \ - will_return(errstart, false); \ - } \ - else \ - { \ - expect_value(errstart_cold, elevel, (LOG_LEVEL)); \ - expect_any(errstart_cold, domain); \ - will_return_with_sideeffect(errstart_cold, false, &_errfinish_impl, NULL); \ - } \ - -static etcdlib_endpoint_t etcd_endpoints[GP_ETCD_ENDPOINTS_NUM] = {0}; -static etcdlib_endpoint_t etcd_multi_endpoints[GP_ETCD_ENDPOINTS_NUM] = {0}; -static etcdlib_endpoint_t etcd_failover_endpoints[GP_ETCD_ENDPOINTS_NUM] = {0}; -static int etcd_endpoints_num = 0; -static char *fts_dump_file_key = NULL; - -static char* -generateLockName() { - uuid_t uid_account; - char uuid_account[UUID_LEN]; - char lock_name[ETCD_LOCK_LEN]; - memset(lock_name, 0, ETCD_LOCK_LEN); - uuid_generate(uid_account); - uuid_unparse(uid_account, uuid_account); - - uuid_t uid_cluster; - char uuid_cluster[UUID_LEN]; - uuid_generate(uid_cluster); - uuid_unparse(uid_cluster, uuid_cluster); - - memset(lock_name, 0, sizeof(char)*ETCD_LOCK_LEN); - snprintf(lock_name, ETCD_LOCK_LEN, "%s/%s/%s/%s/%s", - METADATA_DIR_PREFIX, NAMESPACE, uuid_account, uuid_cluster, ETCD_LOCK_KEY); - - printf("generateLockName lock key: %s\n", lock_name); - return strdup(lock_name); -} - -static void -simpleWriteTest() { - int res = 0; - char*value = NULL; - - res = etcdlib_set(etcdlib, SIMPLEKEY, SIMPLEVALUE, 0, false); - assert_int_equal(res, 0); - - res = etcdlib_get(etcdlib, SIMPLEKEY, &value, NULL); - assert_int_equal(res, 0); - assert_true(value != NULL); - assert_string_equal(value, SIMPLEVALUE); - - res = etcdlib_del(etcdlib, SIMPLEKEY); - assert_int_equal(res, 0); - - // should not get - res = etcdlib_get(etcdlib, SIMPLEKEY, &value, NULL); - assert_int_not_equal(res, 0); - - if (value) - pfree(value); -} - -static void -directroyWriteTest() { - int res = 0; - char*value = NULL; - - res = etcdlib_set(etcdlib, DIRECTORYKEY, SIMPLEVALUE, 0, false); - assert_int_equal(res, 0); - - res = etcdlib_get(etcdlib, DIRECTORYKEY, &value, NULL); - assert_int_equal(res, 0); - assert_true(value != NULL); - assert_string_equal(value, SIMPLEVALUE); - - res = etcdlib_del(etcdlib, DIRECTORYKEY); - assert_int_equal(res, 0); - - /* should not get */ - res = etcdlib_get(etcdlib, DIRECTORYKEY, &value, NULL); - assert_int_not_equal(res, 0); - - if (value) - pfree(value); -} - -int -simpleLeaseTest() { - int res = 0; - long long lease = 0; - char*value = NULL; - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_set(etcdlib, SIMPLEKEY, SIMPLEVALUE, lease, false); - assert_int_equal(res, 0); - - res = etcdlib_get(etcdlib, SIMPLEKEY, &value, NULL); - assert_int_equal(res, 0); - - if (value && strcmp(value, SIMPLEVALUE)) { - printf("etcdlib test error: expected testvalue got %s\n", value); - assert_true(false); - } - - sleep(LEASETIMEOUT + 5); - - res = etcdlib_get(etcdlib, SIMPLEKEY, &value, NULL); - assert_int_not_equal(res, 0); - - if (value) - pfree(value); - return res; - } - -int -simpleLockTest() { - int res = 0; - char*lock_key = NULL; - long long lease = 0; - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, SIMPLELOCK, lease, &lock_key); - assert_int_equal(res, 0); - - sleep(LEASETIMEOUT/2); - - res = etcdlib_unlock(etcdlib, lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); - return res; -} - -int -simpleCheckLeaderTest() { - int res = 0; - bool isLeader = false; - bool failover_result = false; - - res = etcdlib_get_leader(etcdlib, &isLeader); - assert_int_equal(res, 0); - - if (isLeader) { - printf("simpleCheckLeaderTest already found leader node.\n"); - assert_int_equal(isLeader, true); - } else { - printf("simpleCheckLeaderTest already not found leader node, try failover.\n"); - failover_result = test_etcd_failover(&etcdlib); - assert_true(failover_result); - } - - return res; -} - -int -metaLockTest() { - int res = 0; - char* lock_key = NULL; - long long lease = 0; - char *lock_name = generateLockName(); - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - sleep(LEASETIMEOUT/2); - - res = etcdlib_unlock(etcdlib, lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); - return res; -} - -int -metaTimeoutLockTest() { - int res = 0; - char* lock_key = NULL; - long long lease = 0; - char *lock_name = generateLockName(); - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - sleep(LEASETIMEOUT*2); - - lease = 0; - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); - return res; -} - -static void* -etcdLockRenewLease(char *lock_name) { - int rc = 0; - int rc_lock = 0; - long long lease = 0; - int res; - char* lock_key = NULL; - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - printf("etcdLockRenewLease successfully retrived lock.\n"); - - int loop = 0; - while(loop <= 2*LEASETIMEOUT) { - pthread_mutex_lock(&lease_lock); - res = etcdlib_renew_lease(etcdlib, lease); - pthread_mutex_unlock(&lease_lock); - assert_int_equal(res, 0); - loop++; - sleep(1); - } - - res = etcdlib_unlock(etcdlib, lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); -} - -static void* -etcdLockRenewTryLock(char *lock_name) { - int rc = 0; - int rc_lock = 0; - long long lease = 0; - int res; - char* lock_key = NULL; - - sleep(2); - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - pthread_mutex_lock(&lease_lock); - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - pthread_mutex_unlock(&lease_lock); - assert_int_not_equal(res, 0); - - printf("etcdLockRenewTryLock try lock failed as expected.\n"); - - sleep(LEASETIMEOUT + 5); - - lease = 0; - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - pthread_mutex_lock(&lease_lock); - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - pthread_mutex_unlock(&lease_lock); - assert_int_not_equal(res, 0); - - printf("etcdLockRenewTryLock try lock failed again as expected.\n"); - - if (lock_key) - pfree(lock_key); - -} - -int -metaRenewLockTest() { - char *lock_name = generateLockName(); - pthread_t thread1; - pthread_t thread2; - int status = pthread_create(&thread1, NULL, etcdLockRenewLease, (void *)lock_name); - status = pthread_create(&thread2, NULL, etcdLockRenewTryLock, (void *)lock_name); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); -} - -static void* -etcdLockFirst(char *lock_name) { - int rc = 0; - int rc_lock = 0; - long long lease = 0; - int res; - char* lock_key = NULL; - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT*600); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - printf("etcdLockFirst successfully retrived lock.\n"); - sleep(LEASETIMEOUT); - - res = etcdlib_unlock(etcdlib, lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); -} - -static void* -etcdLockFirstTimeout(char *lock_name) { - int rc = 0; - int rc_lock = 0; - long long lease = 0; - int res; - char* lock_key = NULL; - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); -} - -static void* -etcdLockSecond(char *lock_name) { - int rc = 0; - int rc_lock = 0; - long long lease = 0; - int res; - char* lock_key = NULL; - - sleep(2); - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_not_equal(res, 0); - - printf("etcdLockSecond try lock failed as expected.\n"); - - sleep(LEASETIMEOUT + 5); - - lease = 0; - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - assert_int_equal(res, 0); - assert_int_not_equal(lease, 0); - - res = etcdlib_lock(etcdlib, lock_name, lease, &lock_key); - assert_int_equal(res, 0); - - printf("etcdLockSecond successfully retrived lock.\n"); - sleep(1); - - res = etcdlib_unlock(etcdlib, lock_key); - assert_int_equal(res, 0); - - if (lock_key) - pfree(lock_key); -} - -void -metaConcurrentLockTest() { - int status = 0; - char *lock_name = generateLockName(); - pthread_t thread1; - pthread_t thread2; - status = pthread_create(&thread1, NULL, etcdLockFirst, (void *)lock_name); - status = pthread_create(&thread2, NULL, etcdLockSecond, (void *)lock_name); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); -} - -void -metaConcurrentLockTimeoutTest() { - int status = 0; - char *lock_name = generateLockName(); - pthread_t thread1; - pthread_t thread2; - status = pthread_create(&thread1, NULL, etcdLockFirstTimeout, (void *)lock_name); - status = pthread_create(&thread2, NULL, etcdLockSecond, (void *)lock_name); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); -} - -/* temporarilyt disable this case cause the etcdlib watch functionality is currently used, should be added back once necessarily. -void* waitForChange(void*arg) { - int *idx = (int*)arg; - char *action = NULL; - char *prevValue = NULL; - char *value = NULL; - char *rkey = NULL; - long long modifiedIndex; - - printf("Watching for index %d\n", *idx); - - if(etcdlib_watch(etcdlib, "hier/ar", *idx, &action, &prevValue, &value, &rkey, &modifiedIndex) == 0){ - printf(" New value from watch : [%s]%s => %s\n", rkey, prevValue, value); - if(action != NULL) free(action); - if(prevValue != NULL) free(prevValue); - if(rkey != NULL) free(rkey); - if(value != NULL) free(value); - } - - *idx = modifiedIndex+1; - - action = NULL; - prevValue = NULL; - value = NULL; - rkey = NULL; - - if(etcdlib_watch(etcdlib, "hier/ar", *idx, &action, &prevValue, &value, &rkey, &modifiedIndex) == 0){ - printf(" New value from watch : [%s]%s => %s\n", rkey, prevValue, value); - if(action != NULL) free(action); - if(prevValue != NULL) free(prevValue); - if(rkey != NULL) free(rkey); - } - - return value; -} - -int waitforchangetest() { - int res = 0; - char*value = NULL; - - etcdlib_set(etcdlib, "hier/ar/chi/cal", "testvalue1", 5, false); - - int index; - etcdlib_get(etcdlib, "hier/ar/chi/cal", &value, &index); - free(value); - pthread_t waitThread; - index++; - pthread_create(&waitThread, NULL, waitForChange, &index); - sleep(1); - etcdlib_set(etcdlib, "hier/ar/chi/cal", "testvalue2", 5, false); - sleep(1); - etcdlib_set(etcdlib, "hier/ar/chi/cal", "testvalue3", 5, false); - void *resVal = NULL; - printf("joining\n"); - pthread_join(waitThread, &resVal); - if(resVal == NULL || strcmp((char*)resVal,"testvalue3" ) != 0) { - printf("etcdtest::waitforchange1 expected 'testvalue3', got '%s'\n", (char*)resVal); - res = -1; - } - free(resVal); - return res; -} -*/ - -static void -test_init_single_endpoints() { - etcd_endpoints[0].etcd_host = (char *)malloc(sizeof(char)*GP_ETCD_HOSTNAME_LEN); - strncpy(etcd_endpoints[0].etcd_host, ETCD_TEST_HOST, GP_ETCD_HOSTNAME_LEN); - etcd_endpoints[0].etcd_port = ETCD_TEST_PORT; -} - -static void -test_init_multi_endpoints() { - for (int i = 0; i < ETCD_TEST_MULTI_ENDPOINTS_NUM; i++) { - etcd_multi_endpoints[i].etcd_host = (char *)malloc(sizeof(char)*GP_ETCD_HOSTNAME_LEN); - strncpy(etcd_multi_endpoints[i].etcd_host, test_endpoint_list[i], GP_ETCD_HOSTNAME_LEN); - etcd_multi_endpoints[i].etcd_port = ETCD_TEST_PORT; - } -} - -static void -test_init_failover_endpoints() { - for (int i = 0; i < ETCD_TEST_MULTI_ENDPOINTS_NUM+1; i++) { - etcd_failover_endpoints[i].etcd_host = (char *)malloc(sizeof(char)*GP_ETCD_HOSTNAME_LEN); - strncpy(etcd_failover_endpoints[i].etcd_host, test_failover_endpoint_list[i], GP_ETCD_HOSTNAME_LEN); - etcd_failover_endpoints[i].etcd_port = ETCD_TEST_PORT; - } -} - -static void -test_init_endpoints() { - test_init_single_endpoints(); - test_init_multi_endpoints(); - test_init_failover_endpoints(); -} - - -static void -test_etcd_write_test_function(void **state) -{ - etcdlib_endpoint_t *petcd_endpoints = etcd_endpoints; - etcdlib = etcdlib_create(petcd_endpoints, ETCD_TEST_SINGLE_ENDPOINTS_NUM, 0); - simpleWriteTest(); - directroyWriteTest(); - simpleLeaseTest(); - etcdlib_destroy(etcdlib); -} - -static void -test_etcd_lock_test_function(void **state) -{ - etcdlib_endpoint_t *petcd_endpoints = etcd_endpoints; - etcdlib = etcdlib_create(petcd_endpoints, ETCD_TEST_SINGLE_ENDPOINTS_NUM, 0); - simpleLockTest(); - metaLockTest(); - metaTimeoutLockTest(); - metaRenewLockTest(); - metaConcurrentLockTest(); - metaConcurrentLockTimeoutTest(); - etcdlib_destroy(etcdlib); -} - -static void -test_etcd_simple_failover_test_function(void **state) -{ - etcdlib_endpoint_t *petcd_endpoints = etcd_multi_endpoints; - etcdlib = etcdlib_create(petcd_endpoints, ETCD_TEST_MULTI_ENDPOINTS_NUM, 0); - simpleCheckLeaderTest(); - etcdlib_destroy(etcdlib); -} - -static void -test_etcd_failure_failover_test_function(void **state) -{ - int res = 0; - long long lease = 0; - char*value = NULL; - etcdlib_endpoint_t *petcd_endpoints = etcd_failover_endpoints; - etcdlib = etcdlib_create(petcd_endpoints, ETCD_TEST_MULTI_ENDPOINTS_NUM+1, 0); - - res = etcdlib_grant_lease(etcdlib, &lease, LEASETIMEOUT); - - assert_int_equal(res, 0); - etcdlib_destroy(etcdlib); -} - -static void -SetupDataStructures(void **state) -{ - if (NULL == TopMemoryContext) - { - assert_true(NULL == testMemoryContext); - MemoryContextInit(); - - testMemoryContext = AllocSetContextCreate(TopMemoryContext, - "Test Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - - MemoryContextSwitchTo(testMemoryContext); - } - - assert_true(NULL != testMemoryContext && - CurrentMemoryContext == testMemoryContext); -} - -/* - * Cleans up memory by reseting testMemoryContext - */ -static void -TeardownDataStructures(void **state) -{ - assert_true(NULL != testMemoryContext && - CurrentMemoryContext == testMemoryContext); - MemoryContextReset(testMemoryContext); -} - -int -main (int argc, char* argv[]) { - cmockery_parse_arguments(argc, argv); - - test_init_endpoints(); - - const UnitTest tests[] = { - unit_test_setup_teardown(test_etcd_write_test_function, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_etcd_lock_test_function, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_etcd_simple_failover_test_function, SetupDataStructures, TeardownDataStructures), - unit_test_setup_teardown(test_etcd_failure_failover_test_function, SetupDataStructures, TeardownDataStructures) - }; - return run_tests(tests); -} - -#else - -int -main(int argc, char* argv[]) -{ - return 0; -} -#endif \ No newline at end of file diff --git a/src/bin/Makefile b/src/bin/Makefile index 8baea7d1404..582d3f57e59 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -15,7 +15,6 @@ include $(top_builddir)/src/Makefile.global unittest-check: $(MAKE) -C pg_dump/test check - $(MAKE) -C gpfts/test check SUBDIRS = \ initdb \ diff --git a/src/bin/gpfts/test/Makefile b/src/bin/gpfts/test/Makefile deleted file mode 100755 index 74a63d6d417..00000000000 --- a/src/bin/gpfts/test/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -subdir=src/bin/gpfts -top_builddir=../../../.. -etcdlib_obj=$(top_builddir)/src/backend/utils/etcd_lib/etcd.o -include $(top_builddir)/src/Makefile.global - - -TARGETS=gpfts - -override CPPFLAGS+= -I$(top_srcdir)/src/interfaces/libpq -override CPPFLAGS+= -I$(top_srcdir)/src/bin/gpfts - -include $(top_srcdir)/src/Makefile.mock -LDFLAGS := -luuid $(LDFLAGS) - -gpfts.t: gp_fts_test.o $(CMOCKERY_OBJS) ../fts_etcd.o $(etcdlib_obj) $(top_srcdir)/src/fe_utils/log.o - $(CC) $^ $(libpq_pgport) $(LDFLAGS) $(rpath) $(LIBS) $(libpq) -o $@ diff --git a/src/bin/gpfts/test/gp_fts_test.c b/src/bin/gpfts/test/gp_fts_test.c deleted file mode 100644 index 4d6a64c353d..00000000000 --- a/src/bin/gpfts/test/gp_fts_test.c +++ /dev/null @@ -1,184 +0,0 @@ -/*------------------------------------------------------------------------- - * - * gp_fts_test.c - * unit test for separated fts damone testing - * - * - * Portions Copyright (c) 2022-2023, Greenplum Inc. - * - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" -#ifndef USE_INTERNAL_FTS -#include "utils/etcd.h" -#include "lib/stringinfo.h" -#include "nodes/nodes.h" -#include "utils/memutils.h" -#include "cmockery.h" -#include "fts_etcd.h" -#include "fe_utils/log.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SIMPLELOCK "ftslock" -#define SIMPLEHOSTKEY "ftshost" -#define TEST_HOSTNAME "hashdata-fts-0" -#define TEST_NAMESPACE "default" -#define UUID_LEN 37 -#define ETCD_LOCK_LEN 256 - -static char test_single_endpoints[] = GP_ETCD_TEST_SINGLE_ENDPOINTS; -static char test_multi_endpoints[] = GP_ETCD_TEST_MULTI_ENDPOINTS; -static char test_failover_endpoints[] = GP_ETCD_TEST_FAILOVER_ENDPOINTS; -const char *progname = NULL; - -static void -FTSGetLockTest(char *lock_name) { - char *fts_hostname = TEST_HOSTNAME; - char *lock = NULL; - long long lease = 0; - bool res = getFTSLockFromETCD(lock_name, &lock, &lease, FTS_HA_LOCK_LEASE_TIMEOUT_DEFAULT, fts_hostname, SIMPLEHOSTKEY); - assert_true(res); - assert_int_not_equal(lease, 0); -} - -static void -FTSRenewLockLeaseTest(char *lock_name) { - char *fts_hostname = TEST_HOSTNAME; - char *lock = NULL; - long long lease = 0; - bool res = getFTSLockFromETCD(lock_name, &lock, &lease, FTS_HA_LOCK_LEASE_TIMEOUT_DEFAULT, fts_hostname, SIMPLEHOSTKEY); - assert_true(res); - assert_int_not_equal(lease, 0); - int ret = renewFTSLeaseFromETCD(lease, FTS_HA_LOCK_LEASE_TIMEOUT_DEFAULT); - assert_int_equal(ret, 0); -} - -static void -test_fts_lock_function(void **state) -{ - char lock_name[ETCD_LOCK_LEN]; - memset(lock_name, 0, ETCD_LOCK_LEN); - - uuid_t uid_account; - char uuid_account[UUID_LEN]; - uuid_generate(uid_account); - uuid_unparse(uid_account, uuid_account); - - uuid_t uid_cluster; - char uuid_cluster[UUID_LEN]; - uuid_generate(uid_cluster); - uuid_unparse(uid_cluster, uuid_cluster); - - snprintf(lock_name, sizeof(lock_name), "/%s/%s/%s", uuid_account, uuid_cluster, SIMPLELOCK); - - bool res = initETCD(test_single_endpoints, TEST_NAMESPACE, uuid_account, uuid_cluster); - assert_true(res); - FTSGetLockTest(lock_name); - destoryETCD(); -} - -static void -test_fts_lock_multi_function(void **state) -{ - char lock_name[ETCD_LOCK_LEN]; - memset(lock_name, 0, ETCD_LOCK_LEN); - - uuid_t uid_account; - char uuid_account[UUID_LEN]; - uuid_generate(uid_account); - uuid_unparse(uid_account, uuid_account); - - uuid_t uid_cluster; - char uuid_cluster[UUID_LEN]; - uuid_generate(uid_cluster); - uuid_unparse(uid_cluster, uuid_cluster); - - snprintf(lock_name, sizeof(lock_name), "/%s/%s/%s", uuid_account, uuid_cluster, SIMPLELOCK); - - bool res = initETCD(test_multi_endpoints, TEST_NAMESPACE, uuid_account, uuid_cluster); - assert_true(res); - FTSGetLockTest(lock_name); - destoryETCD(); -} - - -static void -test_fts_lock_function_failover_endpoints(void **state) -{ - char lock_name[ETCD_LOCK_LEN]; - memset(lock_name, 0, ETCD_LOCK_LEN); - - uuid_t uid_account; - char uuid_account[UUID_LEN]; - uuid_generate(uid_account); - uuid_unparse(uid_account, uuid_account); - - uuid_t uid_cluster; - char uuid_cluster[UUID_LEN]; - uuid_generate(uid_cluster); - uuid_unparse(uid_cluster, uuid_cluster); - - snprintf(lock_name, sizeof(lock_name), "/%s/%s/%s", uuid_account, uuid_cluster, SIMPLELOCK); - - bool res = initETCD(test_failover_endpoints, TEST_NAMESPACE, uuid_account, uuid_cluster); - assert_true(res); - FTSGetLockTest(lock_name); - destoryETCD(); -} - -static void -test_fts_lock_lease_function(void **state) -{ - char lock_name[ETCD_LOCK_LEN]; - memset(lock_name, 0, ETCD_LOCK_LEN); - - uuid_t uid_account; - char uuid_account[UUID_LEN]; - uuid_generate(uid_account); - uuid_unparse(uid_account, uuid_account); - - uuid_t uid_cluster; - char uuid_cluster[UUID_LEN]; - uuid_generate(uid_cluster); - uuid_unparse(uid_cluster, uuid_cluster); - - snprintf(lock_name, sizeof(lock_name), "/%s/%s/%s", uuid_account, uuid_cluster, SIMPLELOCK); - - bool res = initETCD(test_single_endpoints, TEST_NAMESPACE, uuid_account, uuid_cluster); - assert_true(res); - FTSRenewLockLeaseTest(lock_name); - - destoryETCD(); -} - -int main (int argc, char* argv[]) { - progname = get_progname(argv[0]); - - cmockery_parse_arguments(argc, argv); - - const UnitTest tests[] = { - unit_test(test_fts_lock_function), - unit_test(test_fts_lock_multi_function), - unit_test(test_fts_lock_function_failover_endpoints), - unit_test(test_fts_lock_lease_function) - }; - return run_tests(tests); - return 0; -} -#else -int -main(int argc, char* argv[]) -{ - return 0; -} -#endif diff --git a/src/include/common/etcdutils.h b/src/include/common/etcdutils.h index 599179d1047..f18e458471c 100644 --- a/src/include/common/etcdutils.h +++ b/src/include/common/etcdutils.h @@ -28,9 +28,6 @@ #define GP_ETCD_CLUSTER_ID_DEFAULT "00000000-0000-0000-0000-000000000000" #define GP_ETCD_ENDPOINTS_DEFAULT "localhost:2379" #define GP_CBDB_DEPLOY "onpromise" -#define GP_ETCD_TEST_SINGLE_ENDPOINTS "192.168.180.86:2379" -#define GP_ETCD_TEST_MULTI_ENDPOINTS "192.168.180.86:2379,192.168.180.87:2379,192.168.180.88:2379" -#define GP_ETCD_TEST_FAILOVER_ENDPOINTS "127.0.0.1:2379,192.168.180.86:2379,192.168.180.87:2379,192.168.180.88:2379" typedef struct etcdlib_endpoint { char *etcd_host; diff --git a/src/test/regress/input/gp_tablespace_with_faults.source b/src/test/regress/input/gp_tablespace_with_faults.source index d40225f1855..14af49ce058 100644 --- a/src/test/regress/input/gp_tablespace_with_faults.source +++ b/src/test/regress/input/gp_tablespace_with_faults.source @@ -244,7 +244,10 @@ CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location'; select wait_for_primaries_to_restart(); select force_mirrors_to_catch_up(); select * from list_tablespace_catalog(); +-- start_ignore +-- unstable here select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location'); +-- end_ignore DROP TABLESPACE my_tablespace_for_testing; select cleanup(:content_id_under_test, :'tablespace_location'); diff --git a/src/test/regress/output/gp_tablespace_with_faults.source b/src/test/regress/output/gp_tablespace_with_faults.source index cdff5b1803b..63610c6b95f 100644 --- a/src/test/regress/output/gp_tablespace_with_faults.source +++ b/src/test/regress/output/gp_tablespace_with_faults.source @@ -385,12 +385,14 @@ select * from list_tablespace_catalog(); -1 | pg_global | 1664 (8 rows) +-- start_ignore +-- unstable here select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location'); list_tablespace_dbid_dirs ------------------------------------------------------------------------------------------- {Success,"expected 1 tablespaces","found 1 tablespaces",/tmp/my_tablespace_for_testing/2} (1 row) - +-- end_ignore DROP TABLESPACE my_tablespace_for_testing; ERROR: tablespace "my_tablespace_for_testing" does not exist select cleanup(:content_id_under_test, :'tablespace_location');