the whole shebang

This commit is contained in:
2014-11-25 16:42:40 +01:00
parent 7f74c0613e
commit ab1334c0cf
3686 changed files with 496409 additions and 1 deletions

View File

@@ -0,0 +1,297 @@
<?php
/*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* and is licensed under the MIT license. For more information, see
* <http://www.doctrine-project.org>.
*/
namespace Doctrine\DBAL\Sharding\SQLAzure;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Types\Type;
use Doctrine\DBAL\Schema\Synchronizer\AbstractSchemaSynchronizer;
use Doctrine\DBAL\Schema\Synchronizer\SingleDatabaseSynchronizer;
use Doctrine\DBAL\Schema\Synchronizer\SchemaSynchronizer;
/**
* SQL Azure Schema Synchronizer.
*
* Will iterate over all shards when performing schema operations. This is done
* by partitioning the passed schema into subschemas for the federation and the
* global database and then applying the operations step by step using the
* {@see \Doctrine\DBAL\Schema\Synchronizer\SingleDatabaseSynchronizer}.
*
* @author Benjamin Eberlei <kontakt@beberlei.de>
*/
class SQLAzureFederationsSynchronizer extends AbstractSchemaSynchronizer
{
const FEDERATION_TABLE_FEDERATED = 'azure.federated';
const FEDERATION_DISTRIBUTION_NAME = 'azure.federatedOnDistributionName';
/**
* @var \Doctrine\DBAL\Sharding\SQLAzure\SQLAzureShardManager
*/
private $shardManager;
/**
* @var \Doctrine\DBAL\Schema\Synchronizer\SchemaSynchronizer
*/
private $synchronizer;
/**
* @param \Doctrine\DBAL\Connection $conn
* @param \Doctrine\DBAL\Sharding\SQLAzure\SQLAzureShardManager $shardManager
* @param \Doctrine\DBAL\Schema\Synchronizer\SchemaSynchronizer|null $sync
*/
public function __construct(Connection $conn, SQLAzureShardManager $shardManager, SchemaSynchronizer $sync = null)
{
parent::__construct($conn);
$this->shardManager = $shardManager;
$this->synchronizer = $sync ?: new SingleDatabaseSynchronizer($conn);
}
/**
* {@inheritdoc}
*/
public function getCreateSchema(Schema $createSchema)
{
$sql = array();
list($global, $federation) = $this->partitionSchema($createSchema);
$globalSql = $this->synchronizer->getCreateSchema($global);
if ($globalSql) {
$sql[] = "-- Create Root Federation\n" .
"USE FEDERATION ROOT WITH RESET;";
$sql = array_merge($sql, $globalSql);
}
$federationSql = $this->synchronizer->getCreateSchema($federation);
if ($federationSql) {
$defaultValue = $this->getFederationTypeDefaultValue();
$sql[] = $this->getCreateFederationStatement();
$sql[] = "USE FEDERATION " . $this->shardManager->getFederationName() . " (" . $this->shardManager->getDistributionKey() . " = " . $defaultValue . ") WITH RESET, FILTERING = OFF;";
$sql = array_merge($sql, $federationSql);
}
return $sql;
}
/**
* {@inheritdoc}
*/
public function getUpdateSchema(Schema $toSchema, $noDrops = false)
{
return $this->work($toSchema, function($synchronizer, $schema) use ($noDrops) {
return $synchronizer->getUpdateSchema($schema, $noDrops);
});
}
/**
* {@inheritdoc}
*/
public function getDropSchema(Schema $dropSchema)
{
return $this->work($dropSchema, function($synchronizer, $schema) {
return $synchronizer->getDropSchema($schema);
});
}
/**
* {@inheritdoc}
*/
public function createSchema(Schema $createSchema)
{
$this->processSql($this->getCreateSchema($createSchema));
}
/**
* {@inheritdoc}
*/
public function updateSchema(Schema $toSchema, $noDrops = false)
{
$this->processSql($this->getUpdateSchema($toSchema, $noDrops));
}
/**
* {@inheritdoc}
*/
public function dropSchema(Schema $dropSchema)
{
$this->processSqlSafely($this->getDropSchema($dropSchema));
}
/**
* {@inheritdoc}
*/
public function getDropAllSchema()
{
$this->shardManager->selectGlobal();
$globalSql = $this->synchronizer->getDropAllSchema();
if ($globalSql) {
$sql[] = "-- Work on Root Federation\nUSE FEDERATION ROOT WITH RESET;";
$sql = array_merge($sql, $globalSql);
}
$shards = $this->shardManager->getShards();
foreach ($shards as $shard) {
$this->shardManager->selectShard($shard['rangeLow']);
$federationSql = $this->synchronizer->getDropAllSchema();
if ($federationSql) {
$sql[] = "-- Work on Federation ID " . $shard['id'] . "\n" .
"USE FEDERATION " . $this->shardManager->getFederationName() . " (" . $this->shardManager->getDistributionKey() . " = " . $shard['rangeLow'].") WITH RESET, FILTERING = OFF;";
$sql = array_merge($sql, $federationSql);
}
}
$sql[] = "USE FEDERATION ROOT WITH RESET;";
$sql[] = "DROP FEDERATION " . $this->shardManager->getFederationName();
return $sql;
}
/**
* {@inheritdoc}
*/
public function dropAllSchema()
{
$this->processSqlSafely($this->getDropAllSchema());
}
/**
* @param \Doctrine\DBAL\Schema\Schema $schema
*
* @return array
*/
private function partitionSchema(Schema $schema)
{
return array(
$this->extractSchemaFederation($schema, false),
$this->extractSchemaFederation($schema, true),
);
}
/**
* @param \Doctrine\DBAL\Schema\Schema $schema
* @param boolean $isFederation
*
* @return \Doctrine\DBAL\Schema\Schema
*
* @throws \RuntimeException
*/
private function extractSchemaFederation(Schema $schema, $isFederation)
{
$partitionedSchema = clone $schema;
foreach ($partitionedSchema->getTables() as $table) {
if ($isFederation) {
$table->addOption(self::FEDERATION_DISTRIBUTION_NAME, $this->shardManager->getDistributionKey());
}
if ( $table->hasOption(self::FEDERATION_TABLE_FEDERATED) !== $isFederation) {
$partitionedSchema->dropTable($table->getName());
} else {
foreach ($table->getForeignKeys() as $fk) {
$foreignTable = $schema->getTable($fk->getForeignTableName());
if ($foreignTable->hasOption(self::FEDERATION_TABLE_FEDERATED) !== $isFederation) {
throw new \RuntimeException("Cannot have foreign key between global/federation.");
}
}
}
}
return $partitionedSchema;
}
/**
* Work on the Global/Federation based on currently existing shards and
* perform the given operation on the underlying schema synchronizer given
* the different partitioned schema instances.
*
* @param \Doctrine\DBAL\Schema\Schema $schema
* @param \Closure $operation
*
* @return array
*/
private function work(Schema $schema, \Closure $operation)
{
list($global, $federation) = $this->partitionSchema($schema);
$sql = array();
$this->shardManager->selectGlobal();
$globalSql = $operation($this->synchronizer, $global);
if ($globalSql) {
$sql[] = "-- Work on Root Federation\nUSE FEDERATION ROOT WITH RESET;";
$sql = array_merge($sql, $globalSql);
}
$shards = $this->shardManager->getShards();
foreach ($shards as $shard) {
$this->shardManager->selectShard($shard['rangeLow']);
$federationSql = $operation($this->synchronizer, $federation);
if ($federationSql) {
$sql[] = "-- Work on Federation ID " . $shard['id'] . "\n" .
"USE FEDERATION " . $this->shardManager->getFederationName() . " (" . $this->shardManager->getDistributionKey() . " = " . $shard['rangeLow'].") WITH RESET, FILTERING = OFF;";
$sql = array_merge($sql, $federationSql);
}
}
return $sql;
}
/**
* @return string
*/
private function getFederationTypeDefaultValue()
{
$federationType = Type::getType($this->shardManager->getDistributionType());
switch ($federationType->getName()) {
case Type::GUID:
$defaultValue = '00000000-0000-0000-0000-000000000000';
break;
case Type::INTEGER:
case Type::SMALLINT:
case Type::BIGINT:
$defaultValue = '0';
break;
default:
$defaultValue = '';
break;
}
return $defaultValue;
}
/**
* @return string
*/
private function getCreateFederationStatement()
{
$federationType = Type::getType($this->shardManager->getDistributionType());
$federationTypeSql = $federationType->getSqlDeclaration(array(), $this->conn->getDatabasePlatform());
return "--Create Federation\n" .
"CREATE FEDERATION " . $this->shardManager->getFederationName() . " (" . $this->shardManager->getDistributionKey() . " " . $federationTypeSql ." RANGE)";
}
}

View File

@@ -0,0 +1,242 @@
<?php
/*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* and is licensed under the MIT license. For more information, see
* <http://www.doctrine-project.org>.
*/
namespace Doctrine\DBAL\Sharding\SQLAzure;
use Doctrine\DBAL\Sharding\ShardManager;
use Doctrine\DBAL\Sharding\ShardingException;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Types\Type;
/**
* Sharding using the SQL Azure Federations support.
*
* @author Benjamin Eberlei <kontakt@beberlei.de>
*/
class SQLAzureShardManager implements ShardManager
{
/**
* @var string
*/
private $federationName;
/**
* @var boolean
*/
private $filteringEnabled;
/**
* @var string
*/
private $distributionKey;
/**
* @var string
*/
private $distributionType;
/**
* @var \Doctrine\DBAL\Connection
*/
private $conn;
/**
* @var string
*/
private $currentDistributionValue;
/**
* @param \Doctrine\DBAL\Connection $conn
*
* @throws \Doctrine\DBAL\Sharding\ShardingException
*/
public function __construct(Connection $conn)
{
$this->conn = $conn;
$params = $conn->getParams();
if ( ! isset($params['sharding']['federationName'])) {
throw ShardingException::missingDefaultFederationName();
}
if ( ! isset($params['sharding']['distributionKey'])) {
throw ShardingException::missingDefaultDistributionKey();
}
if ( ! isset($params['sharding']['distributionType'])) {
throw ShardingException::missingDistributionType();
}
$this->federationName = $params['sharding']['federationName'];
$this->distributionKey = $params['sharding']['distributionKey'];
$this->distributionType = $params['sharding']['distributionType'];
$this->filteringEnabled = (isset($params['sharding']['filteringEnabled'])) ? (bool)$params['sharding']['filteringEnabled'] : false;
}
/**
* Gets the name of the federation.
*
* @return string
*/
public function getFederationName()
{
return $this->federationName;
}
/**
* Gets the distribution key.
*
* @return string
*/
public function getDistributionKey()
{
return $this->distributionKey;
}
/**
* Gets the Doctrine Type name used for the distribution.
*
* @return string
*/
public function getDistributionType()
{
return $this->distributionType;
}
/**
* Sets Enabled/Disable filtering on the fly.
*
* @param boolean $flag
*
* @return void
*/
public function setFilteringEnabled($flag)
{
$this->filteringEnabled = (bool)$flag;
}
/**
* {@inheritDoc}
*/
public function selectGlobal()
{
if ($this->conn->isTransactionActive()) {
throw ShardingException::activeTransaction();
}
$sql = "USE FEDERATION ROOT WITH RESET";
$this->conn->exec($sql);
$this->currentDistributionValue = null;
}
/**
* {@inheritDoc}
*/
public function selectShard($distributionValue)
{
if ($this->conn->isTransactionActive()) {
throw ShardingException::activeTransaction();
}
if ($distributionValue === null || is_bool($distributionValue) || !is_scalar($distributionValue)) {
throw ShardingException::noShardDistributionValue();
}
$platform = $this->conn->getDatabasePlatform();
$sql = sprintf(
"USE FEDERATION %s (%s = %s) WITH RESET, FILTERING = %s;",
$platform->quoteIdentifier($this->federationName),
$platform->quoteIdentifier($this->distributionKey),
$this->conn->quote($distributionValue),
($this->filteringEnabled ? 'ON' : 'OFF')
);
$this->conn->exec($sql);
$this->currentDistributionValue = $distributionValue;
}
/**
* {@inheritDoc}
*/
public function getCurrentDistributionValue()
{
return $this->currentDistributionValue;
}
/**
* {@inheritDoc}
*/
public function getShards()
{
$sql = "SELECT member_id as id,
distribution_name as distribution_key,
CAST(range_low AS CHAR) AS rangeLow,
CAST(range_high AS CHAR) AS rangeHigh
FROM sys.federation_member_distributions d
INNER JOIN sys.federations f ON f.federation_id = d.federation_id
WHERE f.name = " . $this->conn->quote($this->federationName);
return $this->conn->fetchAll($sql);
}
/**
* {@inheritDoc}
*/
public function queryAll($sql, array $params = array(), array $types = array())
{
$shards = $this->getShards();
if (!$shards) {
throw new \RuntimeException("No shards found for " . $this->federationName);
}
$result = array();
$oldDistribution = $this->getCurrentDistributionValue();
foreach ($shards as $shard) {
$this->selectShard($shard['rangeLow']);
foreach ($this->conn->fetchAll($sql, $params, $types) as $row) {
$result[] = $row;
}
}
if ($oldDistribution === null) {
$this->selectGlobal();
} else {
$this->selectShard($oldDistribution);
}
return $result;
}
/**
* Splits Federation at a given distribution value.
*
* @param mixed $splitDistributionValue
*
* @return void
*/
public function splitFederation($splitDistributionValue)
{
$type = Type::getType($this->distributionType);
$sql = "ALTER FEDERATION " . $this->getFederationName() . " " .
"SPLIT AT (" . $this->getDistributionKey() . " = " .
$this->conn->quote($splitDistributionValue, $type->getBindingType()) . ")";
$this->conn->exec($sql);
}
}

View File

@@ -0,0 +1,169 @@
<?php
/*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* and is licensed under the MIT license. For more information, see
* <http://www.doctrine-project.org>.
*/
namespace Doctrine\DBAL\Sharding\SQLAzure\Schema;
use Doctrine\DBAL\Schema\Visitor\Visitor;
use Doctrine\DBAL\Schema\Table;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\Column;
use Doctrine\DBAL\Schema\ForeignKeyConstraint;
use Doctrine\DBAL\Schema\Sequence;
use Doctrine\DBAL\Schema\Index;
/**
* Converts a single tenant schema into a multi-tenant schema for SQL Azure
* Federations under the following assumptions:
*
* - Every table is part of the multi-tenant application, only explicitly
* excluded tables are non-federated. The behavior of the tables being in
* global or federated database is undefined. It depends on you selecting a
* federation before DDL statements or not.
* - Every Primary key of a federated table is extended by another column
* 'tenant_id' with a default value of the SQLAzure function
* `federation_filtering_value('tenant_id')`.
* - You always have to work with `filtering=On` when using federations with this
* multi-tenant approach.
* - Primary keys are either using globally unique ids (GUID, Table Generator)
* or you explicitly add the tenent_id in every UPDATE or DELETE statement
* (otherwise they will affect the same-id rows from other tenents as well).
* SQLAzure throws errors when you try to create IDENTIY columns on federated
* tables.
*
* @author Benjamin Eberlei <kontakt@beberlei.de>
*/
class MultiTenantVisitor implements Visitor
{
/**
* @var array
*/
private $excludedTables = array();
/**
* @var string
*/
private $tenantColumnName;
/**
* @var string
*/
private $tenantColumnType = 'integer';
/**
* Name of the federation distribution, defaulting to the tenantColumnName
* if not specified.
*
* @var string
*/
private $distributionName;
/**
* @param array $excludedTables
* @param string $tenantColumnName
* @param string|null $distributionName
*/
public function __construct(array $excludedTables = array(), $tenantColumnName = 'tenant_id', $distributionName = null)
{
$this->excludedTables = $excludedTables;
$this->tenantColumnName = $tenantColumnName;
$this->distributionName = $distributionName ?: $tenantColumnName;
}
/**
* {@inheritdoc}
*/
public function acceptTable(Table $table)
{
if (in_array($table->getName(), $this->excludedTables)) {
return;
}
$table->addColumn($this->tenantColumnName, $this->tenantColumnType, array(
'default' => "federation_filtering_value('". $this->distributionName ."')",
));
$clusteredIndex = $this->getClusteredIndex($table);
$indexColumns = $clusteredIndex->getColumns();
$indexColumns[] = $this->tenantColumnName;
if ($clusteredIndex->isPrimary()) {
$table->dropPrimaryKey();
$table->setPrimaryKey($indexColumns);
} else {
$table->dropIndex($clusteredIndex->getName());
$table->addIndex($indexColumns, $clusteredIndex->getName());
$table->getIndex($clusteredIndex->getName())->addFlag('clustered');
}
}
/**
* @param \Doctrine\DBAL\Schema\Table $table
*
* @return \Doctrine\DBAL\Schema\Index
*
* @throws \RuntimeException
*/
private function getClusteredIndex($table)
{
foreach ($table->getIndexes() as $index) {
if ($index->isPrimary() && ! $index->hasFlag('nonclustered')) {
return $index;
} else if ($index->hasFlag('clustered')) {
return $index;
}
}
throw new \RuntimeException("No clustered index found on table " . $table->getName());
}
/**
* {@inheritdoc}
*/
public function acceptSchema(Schema $schema)
{
}
/**
* {@inheritdoc}
*/
public function acceptColumn(Table $table, Column $column)
{
}
/**
* {@inheritdoc}
*/
public function acceptForeignKey(Table $localTable, ForeignKeyConstraint $fkConstraint)
{
}
/**
* {@inheritdoc}
*/
public function acceptIndex(Table $table, Index $index)
{
}
/**
* {@inheritdoc}
*/
public function acceptSequence(Sequence $sequence)
{
}
}