Search index for deleted

.circleci/config.yml

@@ -2,10 +2,6 @@ version: 2
 defaults:
   init_environemnt: &init_environment
     run: |
-      # SOLR config
-      cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
-      service jetty9 restart || true  # erroring out but does seem to work
-
       # Database Creation
       psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
       createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
@@ -15,53 +11,19 @@ defaults:
 
       # Database Initialization
       ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
+      psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;"
       ckan -c test-core-circle-ci.ini db init
   install_deps: &install_deps
     run: |
       # OS Dependencies
       apt update
-      case $CIRCLE_NODE_INDEX in
-        $NODE_TESTS_CONTAINER)
-        curl -sL https://deb.nodesource.com/setup_10.x | bash -
-        apt install -y nodejs
-        apt install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
-        npm install
-        ;;
-      esac
-      apt install -y postgresql-client solr-jetty openjdk-8-jdk
+      apt install -y postgresql-client
   run_tests: &run_tests
     # Tests Backend, split across containers by segments
     run: |
       mkdir -p ~/junit
-      case $CIRCLE_NODE_INDEX in
-        0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
-        ;;
-        1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
-        ;;
-        2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
-        ;;
-        3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
-        ;;
-      esac
+      python -m pytest -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --splits 4 --group $((CIRCLE_NODE_INDEX+1)) --splitting-algorithm least_duration
 
-  # Tests Frontend, only in one container
-  start_test_server: &start_test_server
-    run:
-      command: |
-        case $CIRCLE_NODE_INDEX in
-          $NODE_TESTS_CONTAINER) ckan -c test-core-circle-ci.ini run
-          ;;
-        esac
-      background: true
-  run_front_tests: &run_front_tests
-    run:
-      command: |
-        case $CIRCLE_NODE_INDEX in
-          $NODE_TESTS_CONTAINER)
-            sleep 5
-            $(npm bin)/cypress run
-          ;;
-        esac
   ckan_env: &ckan_env
     environment:
       CKAN_DATASTORE_POSTGRES_DB: datastore_test
@@ -73,7 +35,6 @@ defaults:
       CKAN_POSTGRES_USER: ckan_default
       CKAN_POSTGRES_PWD: pass
       PGPASSWORD: ckan
-      NODE_TESTS_CONTAINER: 3
       PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
   pg_image: &pg_image
     image: postgres:10
@@ -85,41 +46,19 @@ defaults:
   redis_image: &redis_image
     image: redis:3
     name: ckan-redis
-jobs:
-  test-python-2:
-    docker:
-      - image: python:2-stretch
-        <<: *ckan_env
-      - <<: *pg_image
-      - <<: *redis_image
 
-    parallelism: 4
+  solr_image: &solr_image
+    image: ckan/ckan-solr:master
+    name: ckan-solr
 
-    steps:
-      - checkout
-
-      - <<: *install_deps
-      - run: |
-          # Python Dependencies
-          pip install -r requirement-setuptools.txt
-          pip install -r requirements-py2.txt
-          pip install -r dev-requirements.txt
-          python setup.py develop
-          pip check
-      - <<: *init_environment
-      - <<: *run_tests
-      - store_test_results:
-          path: ~/junit
-      - <<: *start_test_server
-      - <<: *run_front_tests
-      - store_artifacts:
-          path: ~/project/cypress/screenshots
+jobs:
   test-python-3:
     docker:
-      - image: python:3-stretch
+      - image: python:3.7-bullseye
         <<: *ckan_env
       - <<: *pg_image
       - <<: *redis_image
+      - <<: *solr_image
 
     parallelism: 4
 
@@ -140,14 +79,9 @@ jobs:
       - <<: *run_tests
       - store_test_results:
           path: ~/junit
-      - <<: *start_test_server
-      - <<: *run_front_tests
-      - store_artifacts:
-          path: ~/project/cypress/screenshots
       - run: coveralls
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - test-python-2
       - test-python-3

.flake8

@@ -0,0 +1,66 @@
+[flake8]
+exclude =
+        .*
+        ./contrib
+        ./bin
+per-file-ignores =
+                 test_*.py:E501
+extend-exclude =
+               ckan/__init__.py,ckan/config/middleware.py
+               ckan/lib/app_globals.py
+               ckan/lib/cli.py
+               ckan/lib/create_test_data.py
+               ckan/lib/dictization/__init__.py
+               ckan/lib/dictization/model_dictize.py
+               ckan/lib/dictization/model_save.py
+               ckan/lib/email_notifications.py
+               ckan/lib/hash.py
+               ckan/lib/jinja_extensions.py
+               ckan/lib/maintain.py
+               ckan/lib/navl/validators.py
+               ckan/lib/plugins.py
+               ckan/lib/search/__init__.py
+               ckan/lib/search/index.py
+               ckan/lib/search/query.py
+               ckan/logic/action/__init__.py
+               ckan/logic/action/delete.py
+               ckan/logic/action/get.py
+               ckan/logic/action/update.py
+               ckan/logic/auth/create.py
+               ckan/logic/auth/delete.py
+               ckan/logic/auth/get.py
+               ckan/logic/auth/update.py
+               ckan/logic/converters.py
+               ckan/logic/validators.py
+               ckan/model/__init__.py
+               ckan/model/activity.py
+               ckan/model/dashboard.py
+               ckan/model/domain_object.py
+               ckan/model/follower.py
+               ckan/model/group.py
+               ckan/model/group_extra.py
+               ckan/model/license.py
+               ckan/model/meta.py
+               ckan/model/misc.py
+               ckan/model/modification.py
+               ckan/model/package.py
+               ckan/model/package_extra.py
+               ckan/model/package_relationship.py
+               ckan/model/resource.py
+               ckan/model/system_info.py
+               ckan/model/tag.py
+               ckan/model/task_status.py
+               ckan/model/term_translation.py
+               ckan/model/tracking.py
+               ckan/model/user.py
+               ckan/model/vocabulary.py
+               ckan/authz.py
+               ckanext/datastore/logic/action.py
+               ckanext/datastore/tests/test_create.py
+               ckanext/example_idatasetform/plugin.py
+               ckanext/example_itemplatehelpers/plugin.py
+               ckanext/multilingual/plugin.py
+               ckanext/stats/stats.py
+               ckanext/test_tag_vocab_plugin.py
+               doc/conf.py
+               setup.py

.github/ISSUE_TEMPLATE/config.yml

@@ -9,6 +9,6 @@ contact_links:
   - name: Security Issues
     url: mailto:[email protected]
     about: Please report any security related vulnerabilities here.
-  - name: Ideas Repository
-    url: https://github.com/ckan/ideas
-    about: For new feature requests or discussion, please create an issue on the ideas repository.
+  - name: Ideas and Discussion
+    url: https://github.com/ckan/ckan/discussions
+    about: For new feature requests or discussion, please create use the discussion forum.

.github/codeql/codeql-config.yml

@@ -0,0 +1,4 @@
+name: "CKAN CodeQL config"
+
+paths-ignore:
+  - '**/vendor/**'

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,64 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 3 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['python', 'javascript']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        config-file: ./.github/codeql/codeql-config.yml
+        setup-python-dependencies: false
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/cypress.yml

@@ -0,0 +1,70 @@
+name: Cypress
+on: [pull_request]
+env:
+  NODE_VERSION: '16'
+  PYTHON_VERSION: '3.7'
+
+jobs:
+  cypress:
+    runs-on: ubuntu-latest
+    services:
+      ckan-postgres:
+        image: postgres:10
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_USER: ckan_default
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: ckan_test
+
+      ckan-redis:
+        image: redis
+        ports:
+          - 6379:6379
+      ckan-solr:
+        image: ckan/ckan-solr:master
+        ports:
+          - 8983:8983
+
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@localhost/ckan_test
+      CKAN_SOLR_URL: http://localhost:8983/solr/ckan
+      CKAN_REDIS_URL: redis://localhost:6379/1
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install python deps
+        run: pip install -r requirement-setuptools.txt -r requirements.txt -r dev-requirements.txt -e.
+
+      - name: Init environment
+        run: |
+          ckan -c test-core-circle-ci.ini db init
+
+      - name: Run Cypress
+        uses: cypress-io/github-action@v2
+        with:
+          start: ckan -c test-core-circle-ci.ini run
+
+      - uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: cypress/screenshots
+      # Test run video was always captured, so this action uses "always()" condition
+      - uses: actions/upload-artifact@v1
+        if: always()
+        with:
+          name: cypress-videos
+          path: cypress/videos

.github/workflows/flake8.yml

@@ -0,0 +1,19 @@
+name: Lint
+on: [pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        # Stop the build if there are Python syntax errors or undefined names
+        run: flake8 --count --statistics --show-source
+
+      - name: Warnings
+        run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude=""