Revert "Tech Debt: Upgrading boto to boto3 for api & ingester"

Author: Huzaifa-MS

.github/workflows/actions.yml

@@ -3,15 +3,14 @@ on: [push, pull_request]
 
 jobs:
   test-client:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [3.8, 3.9, "3.10", 3.12]
         extras: ["test", "test,queuable,sentry"]
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v2.2.2
         with:
           python-version: ${{ matrix.python }}
       - name: Check out repository code
@@ -21,14 +20,13 @@ jobs:
       - name: Test
         working-directory: ./client
         run: |
-          pip --version
-          pip install --verbose .[${{ matrix.extras }}]
-          python -m pytest --import-mode=importlib ./datalake/tests/
+          pip install -e .[${{ matrix.extras }}]
+          py.test
   test-docker:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - name: Test

.gitignore

@@ -58,6 +58,4 @@ target/
 
 # build artifacts
 version.txt
-
-# Claude Code
-.claude/
+.claude/settings.local.json

Dockerfile

@@ -7,33 +7,29 @@ ENV	LC_ALL C.UTF-8
 
 # TODO: keep requirements in one place
 RUN pip install \
-    'blinker>=1.4' \
-    'boto3==1.35.41' \
-    'botocore==1.35.64' \
-    'click>=5.1' \
-    'datalake<2' \
-    'Flask>=0.10.1' \
-    'flask-swagger>=0.2.14' \
-    'memoized_property>=1.0.1' \
-    'pyinotify>=0.9.4' \
-    'python-dateutil>=2.4.2' \
-    'python-dotenv>=0.1.3' \
-    'pytz>=2015.4' \
-    'raven>=5.0.0' \
-    'requests>=2.5' \
-    'sentry-sdk[flask]>=0.19.5' \
-    'simplejson>=3.3.1' \
-    'six>=1.10.0' \
+    blinker>=1.4 \
+    boto3>=1.1.3 \
+    click>=5.1 \
+    Flask>=0.10.1 \
+    flask-swagger>=0.2.14 \
+    memoized_property>=1.0.1 \
+    python-dateutil>=2.4.2 \
+    python-dotenv>=0.1.3 \
+    pytz>=2015.4 \
+    sentry-sdk[flask]>=0.19.5 \
+    requests>=2.5 \
+    simplejson>=3.3.1 \
+    six>=1.10.0 \
     # test requirements
     'flake8>=2.5.0,<4.1' \
     'freezegun<1' \
-    'moto>=5,<6' \
+    'moto<3' \
     'pytest<8' \
-    'pytest-cov>=2.5.1,<4' \
-    'pytest-xdist' \
     'responses<0.22.0' \
-    'tox>4,<5'
-
+    pyinotify>=0.9.4, \
+    raven>=5.0.0 \
+    'tox>4,<5' \
+    'datalake<2'
 
 RUN mkdir -p /opt/
 COPY . /opt/

Makefile

@@ -12,22 +12,13 @@ dev: docker
 	docker run --rm -it -v $$PWD:/opt --entrypoint /bin/bash $(IMAGE)
 
 test-client: docker
-	docker run --rm -t --entrypoint tox $(IMAGE) -c /opt/client/tox.ini
+	docker run --rm --entrypoint tox $(IMAGE) -c /opt/client/tox.ini
 
 test-ingester: docker
-	docker run --rm -t --entrypoint pytest $(IMAGE) /opt/ingester -svvx
+	docker run --rm --entrypoint py.test $(IMAGE) ingester
 
 test-api: docker
-	docker run --rm -t --entrypoint pytest $(IMAGE) /opt/api -svvx
-
-testp-client: docker
-	docker run --rm -t --entrypoint /bin/bash $(IMAGE) -c "cd /opt/client && pytest -svvx -n auto"
-
-testp-ingester: docker
-	docker run --rm -t --entrypoint pytest $(IMAGE) /opt/ingester -svvx -n auto
-
-testp-api: docker
-	docker run --rm -t --entrypoint pytest $(IMAGE) /opt/api -svvx -n auto
+	docker run --rm --entrypoint py.test $(IMAGE) api
 
 .PHONY: test  # Run the tests
 test:
@@ -36,18 +27,6 @@ test:
 	$(MAKE) test-ingester
 	$(MAKE) test-api
 
-.PHONY: testp  # Run all tests in parallel with pytest-xdist
-testp: docker
-	docker run --rm -t --entrypoint /bin/bash $(IMAGE) -c "\
-		set -e && \
-		echo '==> Running client tests...' && \
-		cd /opt/client && pytest -svvx -n auto && \
-		echo '==> Running ingester tests...' && \
-		cd /opt/ingester && pytest -svvx -n auto && \
-		echo '==> Running API tests...' && \
-		cd /opt/api && pytest -svvx -n auto && \
-		echo '==> All tests passed!'"
-
 .PHONY: push
 push:
 	docker push $(IMAGE)

api/tests/conftest.py

@@ -18,7 +18,7 @@
     ClientError as BotoClientError,
     NoCredentialsError
 )
-from moto import mock_aws
+from moto import mock_dynamodb2
 
 from datalake_api import app as datalake_api
 from datalake.tests import *  # noqa
@@ -33,27 +33,30 @@
 # This will cause moto to fail
 # But more critically, may impact production systems
 # So we test for real credentials and fail hard if they exist
-# Session fixture runs in pytest setup rather than at import time
-@pytest.fixture(scope='session', autouse=True)
-def verify_no_aws_credentials():
-    sts = boto3.client('sts')
-    try:
-        sts.get_caller_identity()
-        pytest.exit("Real AWS credentials detected, aborting", 3)
-    except NoCredentialsError:
-        pass
+sts = boto3.client('sts')
+try:
+    sts.get_caller_identity()
+    pytest.exit("Real AWS credentials detected, aborting", 3)
+except NoCredentialsError:
+    pass  # no credentials are good
 
 
 def get_client():
     from datalake_api import settings
     datalake_api.app.config.from_object(settings)
 
     datalake_api.app.config['TESTING'] = True
-    datalake_api.app.config['AWS_REGION'] = 'us-east-1'
     datalake_api.app.config['AWS_ACCESS_KEY_ID'] = 'abc'
     datalake_api.app.config['AWS_SECRET_ACCESS_KEY'] = '123'
 
-    for a in ('archive_fetcher', 's3_bucket', 'dynamodb'):
+    # TODO: Sigh. The api caches the archive_fetcher and s3_bucket, which is
+    # the right thing. However, because moto<3 still uses httpretty, and
+    # because httpretty wreaks havoc on the python socket code, these cached
+    # parts end up in a bad state after their first use. The right thing to do
+    # here is to upgrade moto. But for that we will also have to move
+    # everything from boto to boto3. This is a near-term goal. But first lets
+    # get everything off of python2.
+    for a in ('archive_fetcher', 's3_bucket'):
         try:
             delattr(datalake_api.app, a)
         except AttributeError:
@@ -68,15 +71,15 @@ def client():
 
 @pytest.fixture
 def dynamodb(request):
-    mock = mock_aws()
+    mock = mock_dynamodb2()
     mock.start()
 
     def tear_down():
         mock.stop()
     request.addfinalizer(tear_down)
 
     return boto3.resource('dynamodb',
-                          region_name='us-east-1',
+                          region_name='us-west-2',
                           aws_secret_access_key='123',
                           aws_access_key_id='abc')
 

client/datalake/tests/conftest.py

@@ -20,7 +20,7 @@
 
 
 try:
-    from moto import mock_aws
+    from moto import mock_s3
     import boto3
     from six.moves.urllib.parse import urlparse
     import json
@@ -125,8 +125,24 @@ def get_tmpfile(content):
 
 
 @pytest.fixture
-def s3_connection():
-    with mock_aws():
+def aws_connector(request):
+
+    def create_connection(mocker, connector):
+        mock = mocker()
+        mock.start()
+
+        def tear_down():
+            mock.stop()
+        request.addfinalizer(tear_down)
+
+        return connector()
+
+    return create_connection
+
+
+@pytest.fixture
+def s3_connection(aws_connector):
+    with mock_s3():
         yield boto3.resource('s3')
 
 

client/pyproject.toml

@@ -21,8 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
-    'boto3==1.35.41',
-    'botocore==1.35.64',
+    'boto3>=1.9.68',
     'memoized_property>=1.0.1',
     'pyblake2>=0.9.3; python_version<"3.6"',
     'click>=4.1',
@@ -38,7 +37,7 @@ dynamic = ["version"]
 test = [
     'pytest<8.0.0',
     'pytest-cov>=2.5.1,<4',
-    'moto[s3]>5,<6',
+    'moto[s3]>4,<5',
     'twine<4.0.0',
     'pip>=20.0.0,<22.0.0',
     'wheel<0.38.0',
@@ -74,7 +73,7 @@ distance-dirty = "{base_version}+{distance}.{vcs}{rev}.dirty"
 # Example formatted version: 1.2.3+42.ge174a1f.dirty
 
 [tool.pytest.ini_options]
-addopts = "--cov=datalake --cov-config .coveragerc"
+addopts = "--cov=planet.mc_client --cov-config .coveragerc"
 markers = [
   "slow: marks tests as slow (deselect with '-m \"not slow\"')"
 ]

ingester/datalake_ingester/queue.py

@@ -13,7 +13,7 @@
 # the License.
 
 from memoized_property import memoized_property
-import boto3
+import boto.sqs
 import simplejson as json
 import logging
 import os
@@ -39,13 +39,16 @@ def set_handler(self, h):
         self.handler = h
 
     @memoized_property
-    def _connection(self):
-        region = os.environ.get('AWS_REGION', 'us-east-1')
-        return boto3.resource('sqs', region_name=region)
+    def _queue(self):
+        return self._connection.get_queue(self.queue_name)
 
     @memoized_property
-    def _queue(self):
-        return self._connection.get_queue_by_name(QueueName=self.queue_name)
+    def _connection(self):
+        region = os.environ.get('AWS_REGION')
+        if region:
+            return boto.sqs.connect_to_region(region)
+        else:
+            return boto.connect_sqs()
 
     _LONG_POLL_TIMEOUT = 20
 
@@ -54,19 +57,17 @@ def drain(self, timeout=None):
         '''
         long_poll_timeout = timeout or self._LONG_POLL_TIMEOUT
         while True:
-            messages = self._queue.receive_messages(
-                MaxNumberOfMessages=1,
-                WaitTimeSeconds=long_poll_timeout
-            )
-            if not messages:
+            raw_msg = self._queue.read(wait_time_seconds=long_poll_timeout)
+            if raw_msg is None:
                 if timeout:
                     return
                 else:
                     continue
-            self._handle_raw_message(messages[0])
+            self._handle_raw_message(raw_msg)
 
     def _handle_raw_message(self, raw_msg):
-        raw = raw_msg.body.replace('\n', ' ')
+        # eliminate newlines in raw message so it all logs to one line
+        raw = raw_msg.get_body().replace('\n', ' ')
         if not self.handler:
             self.logger.error('NO HANDLER CONFIGURED: %s', raw)
             return
@@ -75,4 +76,4 @@ def _handle_raw_message(self, raw_msg):
         msg = json.loads(raw)
 
         self.handler(msg)
-        raw_msg.delete()
+        self._queue.delete_message(raw_msg)

ingester/datalake_ingester/reporter.py

@@ -1,4 +1,4 @@
-import boto3
+import boto.sns
 import simplejson as json
 import logging
 from memoized_property import memoized_property
@@ -25,10 +25,13 @@ def _log_name(self):
 
     @memoized_property
     def _connection(self):
-        region = os.environ.get('AWS_REGION', 'us-east-1')
-        return boto3.client('sns', region_name=region)
+        region = os.environ.get('AWS_REGION')
+        if region:
+            return boto.sns.connect_to_region(region)
+        else:
+            return boto.connect_sns()
 
     def report(self, ingestion_report):
         message = json.dumps(ingestion_report)
         self.logger.info('REPORTING: %s', message)
-        self._connection.publish(TopicArn=self.report_key, Message=message)
+        self._connection.publish(topic=self.report_key, message=message)