let's do this (run tests on windows) (#1889)

Run tests on Windows in Github CI

Author: owais

.github/workflows/test.yml

@@ -10,7 +10,7 @@ env:
   # Otherwise, set variable to the commit of your branch on
   # opentelemetry-python-contrib which is compatible with these Core repo
   # changes.
-  CONTRIB_REPO_SHA: 2a92e255f7595024242e45c0050c8f3de7140b6b
+  CONTRIB_REPO_SHA: dde62cebffe519c35875af6d06fae053b3be65ec
 
 jobs:
   build:
@@ -20,15 +20,15 @@ jobs:
       py37: 3.7
       py38: 3.8
       py39: 3.9
-      pypy3: pypy3
+      pypy3: pypy-3.7
       RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
       matrix:
         python-version: [ py36, py37, py38, py39, pypy3 ]
         package: ["instrumentation", "core", "exporter", "propagator"]
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04, windows-2019 ]
     steps:
       - name: Checkout Core Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v2
@@ -42,14 +42,22 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ env[matrix.python-version] }}
+          architecture: 'x64'
       - name: Install tox
         run: pip install -U tox-factor
       - name: Cache tox environment
         # Preserves .tox directory between runs for faster installs
         uses: actions/cache@v2
         with:
-          path: .tox
-          key: tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v2-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core
+      # tox fails on windows and Python3.6 when tox dir is reused between builds so we remove it
+      - name: fix for windows + py3.6
+        if: ${{ matrix.os == 'windows-2019' && matrix.python-version == 'py36' }} 
+        shell: pwsh
+        run: Remove-Item .\.tox\ -Force -Recurse -ErrorAction Ignore
       - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- --benchmark-json=${{ env.RUN_MATRIX_COMBINATION }}-benchmark.json
       - name: Find and merge benchmarks
@@ -81,7 +89,7 @@ jobs:
       matrix:
         tox-environment: [ "docker-tests", "lint", "docs", "mypy", "mypyinstalled", "tracecontext" ]
     name: ${{ matrix.tox-environment }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout Core Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v2
@@ -95,14 +103,17 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
+          architecture: 'x64'
       - name: Install tox
         run: pip install -U tox
       - name: Cache tox environment
         # Preserves .tox directory between runs for faster installs
         uses: actions/cache@v2
         with:
-          path: .tox
-          key: tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v2-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core
       - name: run tox
         run: tox -e ${{ matrix.tox-environment }}
   contrib-build:
@@ -119,7 +130,7 @@ jobs:
       matrix:
         python-version: [ py36, py37, py38, py39, pypy3 ]
         package: ["instrumentation", "exporter"]
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04]
     steps:
       - name: Checkout Contrib Repo @ SHA - ${{ env.CONTRIB_REPO_SHA }}
         uses: actions/checkout@v2
@@ -135,14 +146,17 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ env[matrix.python-version] }}
+          architecture: 'x64'
       - name: Install tox
         run: pip install -U tox-factor
       - name: Cache tox environment
         # Preserves .tox directory between runs for faster installs
         uses: actions/cache@v2
         with:
-          path: .tox
-          key: tox-cache-${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-contrib
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v2-tox-cache-${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-contrib
       - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }}
   contrib-misc:
@@ -151,7 +165,7 @@ jobs:
       matrix:
         tox-environment: [ "docker-tests"]
     name: ${{ matrix.tox-environment }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout Contrib Repo @ SHA - ${{ env.CONTRIB_REPO_SHA }}
         uses: actions/checkout@v2
@@ -167,13 +181,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
+          architecture: 'x64'
       - name: Install tox
         run: pip install -U tox
       - name: Cache tox environment
         # Preserves .tox directory between runs for faster installs
         uses: actions/cache@v2
         with:
-          path: .tox
-          key: tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-contrib
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v2-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-contrib
       - name: run tox
         run: tox -e ${{ matrix.tox-environment }}

docs/getting_started/flask_example.py

@@ -44,4 +44,4 @@ def hello():
     return "hello"
 
 
-app.run(debug=True, port=5000)
+app.run(port=5000)

shim/opentelemetry-opentracing-shim/tests/testbed/test_common_request_handler/test_asyncio.py

@@ -129,9 +129,9 @@ async def do_task():
         spans = self.tracer.finished_spans()
         self.assertEqual(len(spans), 3)
 
-        spans = sorted(spans, key=lambda x: x.start_time)
         parent_span = get_one_by_operation_name(spans, "parent")
         self.assertIsNotNone(parent_span)
 
-        self.assertIsChildOf(spans[1], parent_span)
-        self.assertIsNotChildOf(spans[2], parent_span)
+        spans = [span for span in spans if span != parent_span]
+        self.assertIsChildOf(spans[0], parent_span)
+        self.assertIsNotChildOf(spans[1], parent_span)

shim/opentelemetry-opentracing-shim/tests/testbed/test_common_request_handler/test_threads.py

@@ -115,5 +115,7 @@ def test_bad_solution_to_set_parent(self):
         parent_span = get_one_by_operation_name(spans, "parent")
         self.assertIsNotNone(parent_span)
 
-        self.assertIsChildOf(spans[1], parent_span)
-        self.assertIsChildOf(spans[2], parent_span)
+        spans = [s for s in spans if s != parent_span]
+        self.assertEqual(len(spans), 2)
+        for span in spans:
+            self.assertIsChildOf(span, parent_span)

tests/util/src/opentelemetry/test/test_base.py

@@ -46,6 +46,11 @@ def tearDownClass(cls):
     def setUp(self):
         self.memory_exporter.clear()
 
+    def get_finished_spans(self):
+        return FinishedTestSpans(
+            self, self.memory_exporter.get_finished_spans()
+        )
+
     def assertEqualSpanInstrumentationInfo(self, span, module):
         self.assertEqual(span.instrumentation_info.name, module.__name__)
         self.assertEqual(span.instrumentation_info.version, module.__version__)
@@ -56,6 +61,12 @@ def assertSpanHasAttributes(self, span, attributes):
             self.assertEqual(val, span.attributes[key])
 
     def sorted_spans(self, spans):  # pylint: disable=R0201
+        """
+        Sorts spans by span creation time.
+
+        Note: This method should not be used to sort spans in a deterministic way as the
+        order depends on timing precision provided by the platform.
+        """
         return sorted(
             spans,
             key=lambda s: s._start_time,  # pylint: disable=W0212
@@ -91,3 +102,23 @@ def disable_logging(highest_level=logging.CRITICAL):
             yield
         finally:
             logging.disable(logging.NOTSET)
+
+
+class FinishedTestSpans(list):
+    def __init__(self, test, spans):
+        super().__init__(spans)
+        self.test = test
+
+    def by_name(self, name):
+        for span in self:
+            if span.name == name:
+                return span
+        self.test.fail("Did not find span with name {}".format(name))
+        return None
+
+    def by_attr(self, key, value):
+        for span in self:
+            if span.attributes.get(key) == value:
+                return span
+        self.test.fail("Did not find span with attrs {}={}".format(key, value))
+        return None