PHPLIB-622 Versioned MongoDB API for Drivers (#816)

* Support new serverParameters runOnRequirement in unified test runner

* Sync unified-test-format tests

* Run versioned API spec tests

This syncs the versioned API spec tests to commit a49cbdf605176a7035c0343779170ce412cc3f0e

* Add helper to create client and manager objects

* Add build variant to test with requireApiVersion=true

* Address code review feedback

* Don't export API_VERSION env variable

Author: alcaeus

.evergreen/config.yml

@@ -176,7 +176,7 @@ functions:
       params:
         script: |
           ${PREPARE_SHELL}
-          MONGODB_VERSION=${VERSION} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+          MONGODB_VERSION=${VERSION} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} REQUIRE_API_VERSION=${REQUIRE_API_VERSION} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
       params:
@@ -215,7 +215,7 @@ functions:
           export GCP_EMAIL="${client_side_encryption_gcp_email}"
           export GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
           export PATH="${PHP_PATH}/bin:$PATH"
-          PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+          API_VERSION=${API_VERSION} PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run atlas data lake test":
      - command: shell.exec
@@ -357,6 +357,17 @@ tasks:
         - func: "bootstrap mongohoused"
         - func: "run atlas data lake test"
 
+    - name: "test-requireApiVersion"
+      tags: ["versioned_api"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            TOPOLOGY: "server"
+            AUTH: "auth"
+            REQUIRE_API_VERSION: "yes"
+        - func: "run tests"
+          vars:
+            API_VERSION: "1"
 
 # }}}
 
@@ -609,3 +620,10 @@ buildvariants:
   run_on: rhel70
   tasks:
     - name: "test-atlas-data-lake"
+
+- matrix_name: "test-requireApiVersion"
+  matrix_spec: { "php-edge-versions": "latest-stable", "versions": "latest", "driver-versions": "latest" }
+  display_name: "Versioned API - ${versions}"
+  run_on: rhel70
+  tasks:
+    - name: "test-requireApiVersion"

.evergreen/run-tests.sh

@@ -12,6 +12,7 @@ AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
 MONGODB_URI=${MONGODB_URI:-}
 TESTS=${TESTS:-}
+API_VERSION=${API_VERSION:-}
 IS_MATRIX_TESTING=${IS_MATRIX_TESTING:-}
 
 # For matrix testing, we have to determine the correct driver version

tests/ClientFunctionalTest.php

@@ -28,7 +28,7 @@ private function doSetUp()
     {
         parent::setUp();
 
-        $this->client = new Client(static::getUri());
+        $this->client = static::createTestClient();
         $this->client->dropDatabase($this->getDatabaseName());
     }
 

tests/DocumentationExamplesTest.php

@@ -4,7 +4,6 @@
 
 use MongoDB\BSON\ObjectId;
 use MongoDB\BSON\UTCDateTime;
-use MongoDB\Client;
 use MongoDB\Database;
 use MongoDB\Driver\Cursor;
 use MongoDB\Driver\Exception\Exception;
@@ -1270,7 +1269,7 @@ public function testTransactions_intro_example_1()
 
         $this->assertNotNull('This test intentionally performs no assertions');
 
-        $client = new Client(static::getUri());
+        $client = static::createTestClient();
 
         /* The WC is required: https://docs.mongodb.com/manual/core/transactions/#transactions-and-locks */
         $client->hr->dropCollection('employees', ['writeConcern' => new WriteConcern('majority')]);
@@ -1445,7 +1444,7 @@ public function testTransactions_retry_example_3()
 
         $this->assertNotNull('This test intentionally performs no assertions');
 
-        $client = new Client(static::getUri());
+        $client = static::createTestClient();
 
         /* The WC is required: https://docs.mongodb.com/manual/core/transactions/#transactions-and-locks */
         $client->hr->dropCollection('employees', ['writeConcern' => new WriteConcern('majority')]);
@@ -1470,7 +1469,7 @@ public function testCausalConsistency()
         $this->assertNotNull('This test intentionally performs no assertions');
 
         // Prep
-        $client = new Client(static::getUri());
+        $client = static::createTestClient();
         $items = $client->selectDatabase(
             'test',
             [ 'writeConcern' => new WriteConcern(WriteConcern::MAJORITY) ]

tests/FunctionalTestCase.php

@@ -4,12 +4,14 @@
 
 use InvalidArgumentException;
 use MongoDB\BSON\ObjectId;
+use MongoDB\Client;
 use MongoDB\Driver\Command;
 use MongoDB\Driver\Exception\CommandException;
 use MongoDB\Driver\Manager;
 use MongoDB\Driver\Query;
 use MongoDB\Driver\ReadPreference;
 use MongoDB\Driver\Server;
+use MongoDB\Driver\ServerApi;
 use MongoDB\Driver\WriteConcern;
 use MongoDB\Operation\CreateCollection;
 use MongoDB\Operation\DatabaseCommand;
@@ -21,6 +23,7 @@
 use function count;
 use function current;
 use function explode;
+use function getenv;
 use function implode;
 use function is_array;
 use function is_object;
@@ -51,7 +54,7 @@ private function doSetUp()
     {
         parent::setUp();
 
-        $this->manager = new Manager(static::getUri());
+        $this->manager = static::createTestManager();
         $this->configuredFailPoints = [];
     }
 
@@ -62,6 +65,16 @@ private function doTearDown()
         parent::tearDown();
     }
 
+    public static function createTestClient(string $uri = null, array $options = [], array $driverOptions = []) : Client
+    {
+        return new Client($uri ?? static::getUri(), $options, static::appendServerApiOption($driverOptions));
+    }
+
+    public static function createTestManager(string $uri = null, array $options = [], array $driverOptions = []) : Manager
+    {
+        return new Manager($uri ?? static::getUri(), $options, static::appendServerApiOption($driverOptions));
+    }
+
     public static function getUri($allowMultipleMongoses = false)
     {
         $uri = parent::getUri();
@@ -86,7 +99,7 @@ public static function getUri($allowMultipleMongoses = false)
             return $uri;
         }
 
-        $manager = new Manager($uri);
+        $manager = static::createTestManager($uri);
         if ($manager->selectServer(new ReadPreference(ReadPreference::RP_PRIMARY))->getType() !== Server::TYPE_MONGOS) {
             return $uri;
         }
@@ -428,6 +441,15 @@ protected function skipIfTransactionsAreNotSupported()
         }
     }
 
+    private static function appendServerApiOption(array $driverOptions) : array
+    {
+        if (getenv('API_VERSION') && ! isset($driverOptions['serverApi'])) {
+            $driverOptions['serverApi'] = new ServerApi(getenv('API_VERSION'));
+        }
+
+        return $driverOptions;
+    }
+
     /**
      * Disables any fail points that were configured earlier in the test.
      *

tests/Operation/FindAndModifyFunctionalTest.php

@@ -3,7 +3,6 @@
 namespace MongoDB\Tests\Operation;
 
 use MongoDB\Driver\BulkWrite;
-use MongoDB\Driver\Manager;
 use MongoDB\Driver\ReadPreference;
 use MongoDB\Model\BSONDocument;
 use MongoDB\Operation\FindAndModify;
@@ -17,7 +16,7 @@ class FindAndModifyFunctionalTest extends FunctionalTestCase
      */
     public function testManagerReadConcernIsOmitted()
     {
-        $manager = new Manager(static::getUri(), ['readConcernLevel' => 'majority']);
+        $manager = static::createTestManager(null, ['readConcernLevel' => 'majority']);
         $server = $manager->selectServer(new ReadPreference(ReadPreference::RP_PRIMARY));
 
         (new CommandObserver())->observe(

tests/Operation/WatchFunctionalTest.php

@@ -11,7 +11,6 @@
 use MongoDB\Driver\Exception\ConnectionTimeoutException;
 use MongoDB\Driver\Exception\LogicException;
 use MongoDB\Driver\Exception\ServerException;
-use MongoDB\Driver\Manager;
 use MongoDB\Driver\Monitoring\CommandSucceededEvent;
 use MongoDB\Driver\ReadPreference;
 use MongoDB\Driver\WriteConcern;
@@ -166,7 +165,7 @@ public function testNextResumesAfterConnectionException()
         /* In order to trigger a dropped connection, we'll use a new client with
          * a socket timeout that is less than the change stream's maxAwaitTimeMS
          * option. */
-        $manager = new Manager(static::getUri(), ['socketTimeoutMS' => 50]);
+        $manager = static::createTestManager(null, ['socketTimeoutMS' => 50]);
         $primaryServer = $manager->selectServer(new ReadPreference(ReadPreference::RP_PRIMARY));
 
         $operation = new Watch($manager, $this->getDatabaseName(), $this->getCollectionName(), [], $this->defaultOptions);

tests/SpecTests/AtlasDataLakeSpecTest.php

@@ -2,7 +2,6 @@
 
 namespace MongoDB\Tests\SpecTests;
 
-use MongoDB\Client;
 use MongoDB\Driver\Command;
 use MongoDB\Driver\Cursor;
 use MongoDB\Tests\CommandObserver;
@@ -135,7 +134,7 @@ public function testKillCursors()
 
         (new CommandObserver())->observe(
             function () {
-                $client = new Client(static::getUri());
+                $client = static::createTestClient();
                 $client->test->driverdata->find([], ['batchSize' => 2, 'limit' => 3]);
             },
             function (array $event) use (&$cursorId, &$cursorNamespace) {
@@ -205,7 +204,7 @@ public function testConnectWithoutAuth()
 
         $uri = $parts['scheme'] . '://' . $parts['host'] . $port . $path . $query;
 
-        $client = new Client($uri);
+        $client = static::createTestClient($uri);
         $cursor = $client->selectDatabase($this->getDatabaseName())->command(['ping' => 1]);
 
         $this->assertInstanceOf(Cursor::class, $cursor);
@@ -217,7 +216,7 @@ public function testConnectWithoutAuth()
      */
     public function testConnectwithSCRAMSHA1()
     {
-        $client = new Client(static::getUri(), ['authMechanism' => 'SCRAM-SHA-1']);
+        $client = static::createTestClient(null, ['authMechanism' => 'SCRAM-SHA-1']);
         $cursor = $client->selectDatabase($this->getDatabaseName())->command(['ping' => 1]);
 
         $this->assertInstanceOf(Cursor::class, $cursor);
@@ -229,7 +228,7 @@ public function testConnectwithSCRAMSHA1()
      */
     public function testConnectwithSCRAMSHA256()
     {
-        $client = new Client(static::getUri(), ['authMechanism' => 'SCRAM-SHA-256']);
+        $client = static::createTestClient(null, ['authMechanism' => 'SCRAM-SHA-256']);
         $cursor = $client->selectDatabase($this->getDatabaseName())->command(['ping' => 1]);
 
         $this->assertInstanceOf(Cursor::class, $cursor);

tests/SpecTests/ClientSideEncryptionSpecTest.php

@@ -5,7 +5,6 @@
 use Closure;
 use MongoDB\BSON\Binary;
 use MongoDB\BSON\Int64;
-use MongoDB\Client;
 use MongoDB\Collection;
 use MongoDB\Driver\ClientEncryption;
 use MongoDB\Driver\Exception\AuthenticationException;
@@ -206,7 +205,7 @@ public function testDataKeyAndDoubleEncryption(string $providerName, $masterKey)
             'keyVaultClient' => $client,
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
         $clientEncryption = $clientEncrypted->createClientEncryption($encryptionOpts);
 
         $commands = [];
@@ -323,7 +322,7 @@ public function testExternalKeyVault($withExternalKeyVault)
         ];
 
         if ($withExternalKeyVault) {
-            $encryptionOpts['keyVaultClient'] = new Client(static::getUri(), ['username' => 'fake-user', 'password' => 'fake-pwd']);
+            $encryptionOpts['keyVaultClient'] = static::createTestClient(null, ['username' => 'fake-user', 'password' => 'fake-pwd']);
         }
 
         $autoEncryptionOpts = $encryptionOpts + [
@@ -332,7 +331,7 @@ public function testExternalKeyVault($withExternalKeyVault)
             ],
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
         $clientEncryption = $clientEncrypted->createClientEncryption($encryptionOpts);
 
         try {
@@ -469,7 +468,7 @@ public function testBSONSizeLimitsAndBatchSplitting(Closure $test)
             'keyVaultClient' => $client,
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         $collection = $clientEncrypted->selectCollection('db', 'coll');
 
@@ -483,7 +482,7 @@ public function testBSONSizeLimitsAndBatchSplitting(Closure $test)
      */
     public function testViewsAreProhibited()
     {
-        $client = new Client(static::getUri());
+        $client = static::createTestClient();
 
         $client->selectCollection('db', 'view')->drop();
         $client->selectDatabase('db')->command(['create' => 'view', 'viewOn' => 'coll']);
@@ -495,7 +494,7 @@ public function testViewsAreProhibited()
             ],
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         try {
             $clientEncrypted->selectCollection('db', 'view')->insertOne(['foo' => 'bar']);
@@ -557,7 +556,7 @@ public function testCorpus($schemaMap = true)
         $corpus = (array) $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus.json'));
         $corpusCopied = [];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
         $clientEncryption = $clientEncrypted->createClientEncryption($encryptionOpts);
 
         $collection = $clientEncrypted->selectCollection('db', 'coll');
@@ -621,7 +620,7 @@ public function testCorpus($schemaMap = true)
      */
     public function testCustomEndpoint(Closure $test)
     {
-        $client = new Client(static::getUri());
+        $client = static::createTestClient();
 
         $clientEncryption = $client->createClientEncryption([
             'keyVaultNamespace' => 'keyvault.datakeys',
@@ -758,7 +757,7 @@ public function testBypassSpawningMongocryptdViaBypassSpawn()
             ],
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         try {
             $clientEncrypted->selectCollection('db', 'coll')->insertOne(['encrypted' => 'test']);
@@ -787,11 +786,11 @@ public function testBypassSpawningMongocryptdViaBypassAutoEncryption()
             ],
         ];
 
-        $clientEncrypted = new Client(static::getUri(), [], ['autoEncryption' => $autoEncryptionOpts]);
+        $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         $clientEncrypted->selectCollection('db', 'coll')->insertOne(['encrypted' => 'test']);
 
-        $clientMongocryptd = new Client('mongodb://localhost:27021');
+        $clientMongocryptd = static::createTestClient('mongodb://localhost:27021');
 
         $this->expectException(ConnectionTimeoutException::class);
         $clientMongocryptd->selectDatabase('db')->command(['isMaster' => true]);