Fix use_ssl: True on Python 3.10

[pquentin]

Rally is a client, so its purpose is to authenticate servers. We
also add tests for the IP and client certs cases to make sure this
change does not break them.

Closes #1484

I also tested this end-to-end with a variation of https://gist.github.com/dliappis/97d013ab6e0e3250fdd6cbfba8e58a5c updated to use Elasticsearch 8.2.0:

cat >.env <<'EOF'
CERTS_DIR=/usr/share/elasticsearch/config/certificates
ELASTIC_PASSWORD=PleaseChangeMe
EOF

cat >instances.yml <<EOF
instances:
  - name: es01
    dns:
      - es01
      - localhost
    ip:
      - 127.0.0.1

  - name: es02
    dns:
      - es02
      - localhost
    ip:
      - 127.0.0.1
EOF

cat >docker-compose.yml <<'EOF'
version: '2.2'

services:
  es01:
    container_name: es01
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    environment:
      - node.name=es01
      - discovery.seed_hosts=es02
      - cluster.initial_master_nodes=es01,es02
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.license.self_generated.type=trial
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.client_authentication=required
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
    volumes:
      - esdata_01:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5

  es02:
    container_name: es02
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    environment:
      - node.name=es02
      - cluster.initial_master_nodes=es01,es02
      - discovery.seed_hosts=es01
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.license.self_generated.type=trial
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.client_authentication=required
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
    volumes:
      - esdata_02:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes: {"esdata_01": {"driver": "local"}, "esdata_02": {"driver": "local"}}
EOF

cat >create-certs.yml <<'EOF'
version: '2.2'

services:
  create_certs:
    container_name: create_certs
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    command: >
      bash -c '
        if [[ ! -d config/certificates/certs ]]; then
          mkdir config/certificates/certs;
        fi;
        if [[ ! -f config/certificates/certs/bundle.zip ]]; then
          echo "certgen!"
          bin/elasticsearch-certgen --silent --in config/certificates/instances.yml --out config/certificates/certs/bundle.zip;
          echo "unzip!"
          unzip config/certificates/certs/bundle.zip -d config/certificates/certs;
        fi;
        chgrp -R 0 config/certificates/certs
      '
    user: ${UID:-1000}
    working_dir: /usr/share/elasticsearch
    volumes: ['.:/usr/share/elasticsearch/config/certificates']
EOF

# Create certs under certs/
docker-compose -f create-certs.yml up
# Start ES with x-pack security enabled requiring full verification of client certs
docker-compose up -d


echo "sleep"
sleep 60

# Confirm things are working as expected with curl
curl --cacert certs/ca/ca.crt --cert certs/es01/es01.crt --key certs/es01/es01.key https://elastic:PleaseChangeMe@127.0.0.1:9200

# Run a race both with hostname and IP
esrally race --track=pmc --target-hosts=localhost:9200 --pipeline=benchmark-only --client-options="use_ssl:true,verify_certs:true,client_cert:'$PWD/certs/es01/es01.crt',client_key:'$PWD/certs/es01/es01.key',ca_certs:'$PWD/certs/ca/ca.crt',basic_auth_user:'elastic',basic_auth_password:'PleaseChangeMe'" --test-mode
esrally race --track=pmc --target-hosts=127.0.0.1:9200 --pipeline=benchmark-only --client-options="use_ssl:true,verify_certs:true,client_cert:'$PWD/certs/es01/es01.crt',client_key:'$PWD/certs/es01/es01.key',ca_certs:'$PWD/certs/ca/ca.crt',basic_auth_user:'elastic',basic_auth_password:'PleaseChangeMe'" --test-mode

[michaelbaamonde]

This looks really nice to me, especially the tests.

The background on the linked issue is also quite interesting. Not a blocker, but if you wouldn't mind including a bit of the context you provided on that issue (e.g. why we need to use SERVER_AUTH as opposed to CLIENT_AUTH) in the commit message when you squash and merge, that would be great.

[pquentin]

Thanks, I provided context in the commit when squashing.