diff --git a/file_catalog/mongo.py b/file_catalog/mongo.py index 9950948..9c2ed1b 100644 --- a/file_catalog/mongo.py +++ b/file_catalog/mongo.py @@ -7,8 +7,8 @@ from typing import Any, Dict, List, Optional, Union, cast from motor.motor_tornado import MotorClient, MotorCursor # type: ignore[import] -import pymongo -from pymongo.results import InsertOneResult +import pymongo # type: ignore[import] +from pymongo.results import InsertOneResult # type: ignore[import] from wipac_telemetry import tracing_tools as wtt from .schema.types import Metadata diff --git a/requirements-dev.txt b/requirements-dev.txt index 8ab6d79..7b3fe0b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -38,8 +38,6 @@ cryptography==40.0.0 # via pyjwt deprecated==1.2.13 # via opentelemetry-api -dnspython==2.3.0 - # via pymongo exceptiongroup==1.1.1 # via pytest flake8==6.0.0 @@ -70,7 +68,7 @@ mccabe==0.7.0 # via flake8 more-itertools==9.1.0 # via wipac-file-catalog (setup.py) -motor==3.1.1 +motor==2.5.1 # via wipac-file-catalog (setup.py) multidict==6.0.4 # via @@ -124,7 +122,7 @@ pyflakes==3.0.1 # via flake8 pyjwt[crypto]==2.5.0 # via wipac-rest-tools -pymongo==4.3.3 +pymongo==3.13.0 # via # motor # wipac-file-catalog (setup.py) diff --git a/requirements.txt b/requirements.txt index e56b3a0..0811ba9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,8 +22,6 @@ cryptography==40.0.0 # via pyjwt deprecated==1.2.13 # via opentelemetry-api -dnspython==2.3.0 - # via pymongo googleapis-common-protos==1.56.2 # via # opentelemetry-exporter-jaeger-proto-grpc @@ -38,7 +36,7 @@ importlib-metadata==6.0.1 # via opentelemetry-api ldap3==2.9.1 # via wipac-file-catalog (setup.py) -motor==3.1.1 +motor==2.5.1 # via wipac-file-catalog (setup.py) opentelemetry-api==1.17.0 # via @@ -76,7 +74,7 @@ pycparser==2.21 # via cffi pyjwt[crypto]==2.5.0 # via wipac-rest-tools -pymongo==4.3.3 +pymongo==3.13.0 # via # motor # wipac-file-catalog (setup.py) diff --git a/resources/enable_profiling.py b/resources/enable_profiling.py index 189c8da..b1527e8 100755 --- a/resources/enable_profiling.py +++ b/resources/enable_profiling.py @@ -1,21 +1,11 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # fmt:off import os -from typing import Any, cast, Dict +import pymongo # type: ignore[import] from pymongo import MongoClient -from pymongo.database import Database - -# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4) -# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL -# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile -OFF = 0 -SLOW_ONLY = 1 -ALL = 2 - -FCDoc = Dict[str, Any] env = { 'TEST_DATABASE_HOST': 'localhost', @@ -30,10 +20,6 @@ else: env[k] = os.environ[k] -test_database_host = str(env['TEST_DATABASE_HOST']) -test_database_port = int(str(env['TEST_DATABASE_PORT'])) -db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog) -# db.set_profiling_level(pymongo.OFF) -# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed -db.command('profile', ALL, filter={'op': 'query'}) +db = MongoClient(host=env['TEST_DATABASE_HOST'], port=env['TEST_DATABASE_PORT']).file_catalog +db.set_profiling_level(pymongo.ALL) print('MongoDB profiling enabled') diff --git a/resources/enable_profiling_future.py b/resources/enable_profiling_future.py new file mode 100755 index 0000000..bea2bb0 --- /dev/null +++ b/resources/enable_profiling_future.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# fmt:off + +import os +from typing import Any, cast, Dict + +from pymongo import MongoClient # type: ignore[import] +from pymongo.database import Database # type: ignore[import] + +# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4) +# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL +# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile +OFF = 0 +SLOW_ONLY = 1 +ALL = 2 + +FCDoc = Dict[str, Any] + +env = { + 'TEST_DATABASE_HOST': 'localhost', + 'TEST_DATABASE_PORT': 27017, +} +for k in env: + if k in os.environ: + if isinstance(env[k], int): + env[k] = int(os.environ[k]) + elif isinstance(env[k], float): + env[k] = float(os.environ[k]) + else: + env[k] = os.environ[k] + +test_database_host = str(env['TEST_DATABASE_HOST']) +test_database_port = int(str(env['TEST_DATABASE_PORT'])) +db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog) +# db.set_profiling_level(pymongo.OFF) +# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed +db.command('profile', ALL, filter={'op': 'query'}) +print('MongoDB profiling enabled') diff --git a/resources/profile_queries.py b/resources/profile_queries.py index ab5049b..13f975c 100755 --- a/resources/profile_queries.py +++ b/resources/profile_queries.py @@ -1,21 +1,12 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # fmt:off +# flake8:noqa import os -from typing import Any, cast, Dict +import pymongo # type: ignore[import] from pymongo import MongoClient -from pymongo.database import Database - -# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4) -# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL -# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile -OFF = 0 -SLOW_ONLY = 1 -ALL = 2 - -FCDoc = Dict[str, Any] env = { 'TEST_DATABASE_HOST': 'localhost', @@ -30,33 +21,25 @@ else: env[k] = os.environ[k] -test_database_host = str(env['TEST_DATABASE_HOST']) -test_database_port = int(str(env['TEST_DATABASE_PORT'])) -db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog) - -# level = db.profiling_level() -# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-profiling-level-is-removed -profile: Dict[str, Any] = db.command('profile', -1) -level = profile['was'] -if level != ALL: +db = MongoClient(host=env['TEST_DATABASE_HOST'], port=env['TEST_DATABASE_PORT']).file_catalog +ret = db.profiling_level() +if ret != pymongo.ALL: raise Exception('profiling disabled') -# db.set_profiling_level(pymongo.OFF) -# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed -db.command('profile', ALL, filter={'op': 'query'}) +db.set_profiling_level(pymongo.OFF) unrealistic_queries = [ {'locations.archive': True}, {'locations.archive': False}, {'locations.archive': None}, {'locations.archive': None, 'run.first_event': {'$lte': 400}, 'run.last_event': {'$gte': 400}}, - # the query to get the queries that we're profiling doesn't count! - {'op': {'$nin': ['command', 'insert']}}, ] bad_queries = [] -ret = db.system.profile.find({'op': {'$nin': ['command', 'insert']}}) +ret = db.system.profile.find({ 'op': { '$nin' : ['command', 'insert'] } }) for query in ret: try: + if 'find' in query['command'] and query['command']['find'] == 'collections': + continue # exclude unrealistic test queries if 'filter' in query['command'] and query['command']['filter'] in unrealistic_queries: continue @@ -64,13 +47,13 @@ print(query) continue if 'IXSCAN' not in query['planSummary']: - bad_queries.append((query['command'], query['planSummary'])) + bad_queries.append((query['command'],query['planSummary'])) except Exception: print(query) raise if bad_queries: - for q, p in bad_queries: + for q,p in bad_queries: print(q) print(p) print('---') diff --git a/resources/profile_queries_future.py b/resources/profile_queries_future.py new file mode 100755 index 0000000..8ad609d --- /dev/null +++ b/resources/profile_queries_future.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# fmt:off + +import os +from typing import Any, cast, Dict + +from pymongo import MongoClient # type: ignore[import] +from pymongo.database import Database # type: ignore[import] + +# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4) +# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL +# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile +OFF = 0 +SLOW_ONLY = 1 +ALL = 2 + +FCDoc = Dict[str, Any] + +env = { + 'TEST_DATABASE_HOST': 'localhost', + 'TEST_DATABASE_PORT': 27017, +} +for k in env: + if k in os.environ: + if isinstance(env[k], int): + env[k] = int(os.environ[k]) + elif isinstance(env[k], float): + env[k] = float(os.environ[k]) + else: + env[k] = os.environ[k] + +test_database_host = str(env['TEST_DATABASE_HOST']) +test_database_port = int(str(env['TEST_DATABASE_PORT'])) +db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog) + +# level = db.profiling_level() +# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-profiling-level-is-removed +profile: Dict[str, Any] = db.command('profile', -1) +level = profile['was'] +if level != ALL: + raise Exception('profiling disabled') +# db.set_profiling_level(pymongo.OFF) +# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed +db.command('profile', ALL, filter={'op': 'query'}) + +unrealistic_queries = [ + {'locations.archive': True}, + {'locations.archive': False}, + {'locations.archive': None}, + {'locations.archive': None, 'run.first_event': {'$lte': 400}, 'run.last_event': {'$gte': 400}}, + # the query to get the queries that we're profiling doesn't count! + {'op': {'$nin': ['command', 'insert']}}, +] + +bad_queries = [] +ret = db.system.profile.find({'op': {'$nin': ['command', 'insert']}}) +for query in ret: + try: + # exclude unrealistic test queries + if 'filter' in query['command'] and query['command']['filter'] in unrealistic_queries: + continue + if 'planSummary' not in query: + print(query) + continue + if 'IXSCAN' not in query['planSummary']: + bad_queries.append((query['command'], query['planSummary'])) + except Exception: + print(query) + raise + +if bad_queries: + for q, p in bad_queries: + print(q) + print(p) + print('---') + raise Exception('Non-indexed queries') + +print('MongoDB profiling OK') diff --git a/setup.cfg b/setup.cfg index e40feb9..e0aba1d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,8 +57,8 @@ packages = find: install_requires = coloredlogs ldap3 - motor - pymongo + motor<3 + pymongo<4 requests requests-futures requests-toolbelt diff --git a/tests/conftest.py b/tests/conftest.py index 27d2aa4..ae4b771 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,8 +12,8 @@ import socket from typing import Any, AsyncGenerator, cast, Dict -from pymongo import MongoClient -from pymongo.errors import ServerSelectionTimeoutError +from pymongo import MongoClient # type: ignore[import] +from pymongo.errors import ServerSelectionTimeoutError # type: ignore[import] import pytest from pytest import MonkeyPatch import pytest_asyncio diff --git a/tests/test_mongo.py b/tests/test_mongo.py index f399566..7d869fa 100644 --- a/tests/test_mongo.py +++ b/tests/test_mongo.py @@ -10,7 +10,7 @@ from file_catalog.mongo import AllKeys, Mongo from motor import MotorCollection # type: ignore[import] -from pymongo.errors import DuplicateKeyError +from pymongo.errors import DuplicateKeyError # type: ignore[import] logger = logging.getLogger(__name__)