Use %tox for testing instead of %pytest

[skip changelog]

python-sentry-sdk.spec

@@ -23,11 +23,6 @@
 
 %bcond network_tests 0
 
-# TODO:
-# sqlalchemy installed during tests causes many failures.
-# It is used by sentry_sdk/db/explain_plan/sqlalchemy.py and is optional, drop it.
-%bcond sqlalchemy_during_tests 0
-
 %global forgeurl https://github.com/getsentry/sentry-python
 Version:        1.39.2
 %global tag %{version}
@@ -84,41 +79,129 @@ Summary:        %{summary}
 
 %description -n python3-sentry-sdk %_description
 
-%global extras_excluded %{shrink:
+%global default_toxenv py%{python3_version}
+
+# List of names of extras & toxenvs included
+%global components %{shrink:
+  aiohttp
+  celery
+  django
+  falcon
+  fastapi
+  opentelemetry
+  pure_eval
+  sqlalchemy
+  starlette
+  tornado
+  %{nil}}
+
+# List of names of extras & toxenvs excluded
+# arq: no_arq
+# beam: no_beam
+# chalice: no_chalice
+# huey: no_huey
+# loguru: no_loguru
+# quart: no_quart
+# sanic: no_sanic
+# starlite: no_starlite
+%global components_excluded %{shrink:
   arq
   beam
   chalice
-  clickhouse-driver
   huey
   loguru
-  %{!?with_opentelemetry_experimental:opentelemetry-experimental}
-  pyspark
   quart
   sanic
   starlite
   %{nil}}
 
+# List of names of extras included (if not present in components)
 %global extras %{shrink:
-  aiohttp
+  %{components}
   asyncpg
   bottle
-  celery
-  django
-  falcon
-  fastapi
   flask
   grpcio
   httpx
-  opentelemetry
-  %{?with_opentelemetry_experimental:opentelemetry-experimental}
-  pure_eval
   pymongo
   rq
-  sqlalchemy
+  %{?with_opentelemetry_experimental:opentelemetry-experimental}
-  starlette
-  tornado
   %{nil}}
 
+# List of names of extras excluded (if not present in components_excluded)
+# clickhouse-driver: no_clickhouse_driver
+# pyspark: no_pyspark
+%global extras_excluded %{shrink:
+  %{components_excluded}
+  clickhouse-driver
+  pyspark
+  %{!?with_opentelemetry_experimental:opentelemetry-experimental}
+  %{nil}}
+
+%define toxenvs_by_components %{expand:%(echo %{components} | sed "s/^/%{toxenv}-/;s/ / %{toxenv}-/g")}
+
+# List of names of toxenvs included (if not present in components)
+%global toxenvs %{shrink:
+  %{toxenvs_by_components}
+  %{toxenv}-common
+  %{toxenv}-cloud_resource_context
+  %{toxenv}-gevent
+  %{nil}
+}
+
+%define toxenvs_excluded_by_components %{expand:%(echo %{components_excluded} | sed "s/^/%{toxenv}-/;s/ / %{toxenv}-/g")}
+
+# List of names of toxenvs excluded (if not present in components_excluded)
+# ariadne: no_ariadne
+# asgi: async_asgi_testclient is unpackaged yet
+# asyncpg: require a local PostgreSQL instance running
+# aws_lambda: aws_lambda requires credentials
+# boto3: require network
+# bottle: werkzeug in Fedora 38 is too new, see: https://github.com/getsentry/sentry-python/issues/1398
+# clickhouse_driver: no_clickhouse_driver
+# flask: werkzeug in Fedora 38 is too new, see: https://github.com/getsentry/sentry-python/issues/1398
+# gcp: python 3.7 only
+# gql: no_gql
+# graphene: graphene is too old (min version: 3.3)
+# grpc: protobuf is too old
+# httpx: require network
+# pymongo: mockupdb is unpackaged because it appears unmaintained
+# pyramid: werkzeug is too new (version < 2.1.0)
+# redis: fakeredis is unpackaged yet
+# rediscluster: fakeredis is unpackaged yet
+# requests: require network
+# rq: fakeredis is unpackaged yet
+# socket: require network
+# starberry: no_strawberry
+# trytond: no_trytond
+%global toxenvs_excluded %{shrink:
+  %{toxenvs_excluded_by_components}
+  %{toxenv}-ariadne
+  %{toxenv}-asgi
+  %{toxenv}-asyncpg
+  %{toxenv}-aws_lambda
+  %{toxenv}-boto3
+  %{toxenv}-bottle
+  %{toxenv}-clickhouse_driver
+  %{toxenv}-flask
+  %{toxenv}-gcp
+  %{toxenv}-gql
+  %{toxenv}-graphene
+  %{toxenv}-grpc
+  %{toxenv}-httpx
+  %{toxenv}-pymongo
+  %{toxenv}-pyramid
+  %{toxenv}-redis
+  %{toxenv}-rediscluster
+  %{toxenv}-rq
+  %{toxenv}-requests
+  %{toxenv}-socket
+  %{toxenv}-strawberry
+  %{toxenv}-trytond
+  %{nil}}
+
+%define toxenvs_csv %{expand:%(echo %{toxenvs} | sed "s/ /,/g")}
+
 %define extras_csv %{expand:%(echo %{extras} | sed "s/ /,/g")}
 
 %pyproject_extras_subpkg -n python3-sentry-sdk %{extras}
@@ -132,8 +215,25 @@ defined_extra=$(echo "%extras_excluded" "%extras" | xargs -n1 | sort -u)
 setup_py_extra=$(cat setup.py | sed -n '/extras_require/,/}/p' | sed 's/    //g' | sed '$ s/.$/\nprint("\\n".join(extras_require))/' | python3 -)
 diff <(echo "$defined_extra") <(echo "$setup_py_extra")
 
+# Unpin all test dependencies to make installation happen
+sed -r -i '/pytest-cov==2\.8\.1/d' test-requirements.txt
+sed -r -i '/tox==3\.7\.0/d' test-requirements.txt
+sed -r -i 's/(executing)<2\.0\.0/\1/' test-requirements.txt
+sed -r -i 's/(jsonschema)==3\.2\.0/\1/' test-requirements.txt
+sed -r -i 's/(pyrsistent)==0\.16\.0/\1/' test-requirements.txt
+sed -r -i 's/(pytest-forked)<=1\.4\.0/\1/' test-requirements.txt
+sed -r -i 's/(pytest-localserver)==0\.5\.1/\1/' test-requirements.txt
+
+sed -r -i 's/psycopg2-binary/psycopg2/' tox.ini
+sed -r -i 's/(pytest)<7\.0\.0/\1/' tox.ini
+sed -r -i 's/(Werkzeug)<2\.1\.0/\1/' tox.ini
+sed -r -i 's/(gevent)>=22\.10\.0, <22\.11\.0/\1/' tox.ini
+
+# newrelic is unpackaged yet, remove it
+sed -r -i '/(newrelic)/d' tox.ini
+
 %generate_buildrequires
-%pyproject_buildrequires -x %{extras_csv}
+%pyproject_buildrequires -x %{extras_csv} -e %{toxenvs_csv}
 
 
 %build
@@ -194,7 +294,6 @@ ignore="${ignore-} --ignore=tests/integrations/socket"
 %endif
 # TODO: investigate
 ignore="${ignore-} --ignore=tests/integrations/fastapi"
-ignore="${ignore-} --ignore=tests/integrations/httpx"
 # require credentials
 ignore="${ignore-} --ignore=tests/integrations/aws_lambda/"
 
@@ -216,9 +315,6 @@ deselect="${deselect-} --deselect=tests/utils/test_contextvars.py"
 deselect="${deselect-} --deselect=tests/test_transport.py::test_transport_works"
 ignore="${ignore-} --ignore=tests/integrations/django"
 
-# async_asgi_testclient is unpackaged yet
-ignore="${ignore-} --ignore=tests/integrations/asgi"
-
 # fakeredis is unpackaged yet
 deselect="${deselect-} --deselect=tests/test_basics.py::test_redis_disabled_when_not_installed"
 ignore="${ignore-} --ignore=tests/integrations/redis"
@@ -239,53 +335,37 @@ ignore="${ignore-} --ignore=tests/integrations/pyramid"
 # newrelic is unpackaged yet
 deselect="${deselect-} --deselect=tests/integrations/celery/test_celery.py::test_newrelic_interference"
 
-# rediscluster is unpackaged yet
-ignore="${ignore-} --ignore=tests/integrations/rediscluster"
-
 # werkzeug in Fedora 38 is too new, see: https://github.com/getsentry/sentry-python/issues/1398
 ignore="${ignore-} --ignore=tests/integrations/bottle"
 ignore="${ignore-} --ignore=tests/integrations/flask"
 
-# disabled extras/integrations
-ignore="${ignore-} --ignore=tests/integrations/ariadne"  # no_ariadne
-ignore="${ignore-} --ignore=tests/integrations/arq"  # no_arq
-ignore="${ignore-} --ignore=tests/integrations/beam"  # no_beam
-ignore="${ignore-} --ignore=tests/integrations/chalice"  # no_chalice
-ignore="${ignore-} --ignore=tests/integrations/clickhouse_driver"  # no_clickhouse_driver
-ignore="${ignore-} --ignore=tests/integrations/gql"  # no_gql
-ignore="${ignore-} --ignore=tests/integrations/huey"  # no_huey
-ignore="${ignore-} --ignore=tests/integrations/loguru"  # no_loguru
-ignore="${ignore-} --ignore=tests/integrations/spark"  # no_pyspark
-ignore="${ignore-} --ignore=tests/integrations/quart"  # no_quart
-ignore="${ignore-} --ignore=tests/integrations/sanic"  # no_sanic
-ignore="${ignore-} --ignore=tests/integrations/starlite"  # no_starlite
-ignore="${ignore-} --ignore=tests/integrations/strawberry"  # no_strawberry
-ignore="${ignore-} --ignore=tests/integrations/trytond"  # no_trytond
-
 ignore="${ignore-} %{!?with_opentelemetry_experimental:--ignore=tests/integrations/opentelemetry/test_experimental.py}"
 
-# Make django testing separate:
+# TODO
-# pytest-django cannot find manage.py, since the layout is custom:
+deselect="${deselect-} --deselect=tests/integrations/threading/test_threading.py::test_circular_references"
-# https://github.com/getsentry/sentry-python/blob/1.39.1/tests/integrations/django/myapp/manage.py
-# is too deep inside, so it expects manual setting of PYTHONPATH:
-# https://pytest-django.readthedocs.io/en/latest/managing_python_path.html
-# If we add . to PYTHONPATH for all tests, how can we be sure that packaged library works fine (we are testing sources)?
-sed -i 's/\[pytest\]/[pytest]\ndjango_find_project = false/' pytest.ini
-PYTHONPATH=. pytest -rsx -s --durations=5 tests/integrations/django
-sed -i '/django_find_project =/D' pytest.ini
-sed -i '/DJANGO_SETTINGS_MODULE =/D' pytest.ini
 
-%if %{without sqlalchemy_during_tests}
+defined_toxenvs=$(echo "%toxenvs_excluded" "%toxenvs" | xargs -n1 | sort -u)
-# Test sqlalchemy separately
+tox_ini_toxenvs=$(cat tox.ini | sed -r -n 's/[[:blank:]]*(.*):[[:blank:]]*TESTPATH=.*/%{default_toxenv}-\1/p' | xargs -n1 | sort -u)
-%pytest -rsx -s --durations=5 tests/integrations/sqlalchemy
+diff <(echo "$defined_toxenvs") <(echo "$tox_ini_toxenvs")
+
+# TODO:
+# sqlalchemy installed during tests causes many failures.
+# It is used by sentry_sdk/db/explain_plan/sqlalchemy.py and is optional, drop it.
+# Test sqlalchemy separately.
+%tox -e %{default_toxenv}-sqlalchemy -- -- ${deselect-} ${ignore-}
 
 # Make `import sqlalchemy` fail
 echo "raise ImportError()" > sqlalchemy.py
 
-ignore="${ignore-} --ignore=tests/integrations/sqlalchemy"
+%define toxenvs_csv_without_sqlalchemy %{expand:%(echo %{toxenvs_csv} | sed "s/%{default_toxenv}-sqlalchemy,//")}
-%endif
 
-%pytest -rsx -s --durations=5 tests/ ${deselect-} ${ignore-}
+# TODO:
+# By some reason, some tests fail because of django plugin, run them separately.
+%define toxenvs_csv_without_django_and_sqlalchemy %{expand:%(echo %{toxenvs_csv} | sed "s/%{default_toxenv}-django,//")}
+no_django="test_link_trace_context_to_error_event or test_on_end_sentry_span or test_on_end_sentry_transaction or test_pruning_old_spans_on_end or test_transaction_style or test_user_information_error or test_circular_references"
+%tox -e %{toxenvs_csv_without_django_and_sqlalchemy} -- -- -p no:django ${deselect-} ${ignore-} tests/integrations/opentelemetry tests/integrations/starlette/test_starlette.py tests/integrations/threading/test_threading.py -k "$no_django"
+
+%tox -e %{toxenvs_csv_without_sqlalchemy} -- -- ${deselect-} ${ignore-} -k "not ($no_django)"
 
 %files -n python3-sentry-sdk -f %{pyproject_files}
 %doc README.md