Apache ShardingSphere 元数据能力增强解读与实战

governance_ds
--metadata (元数据信息)
----sharding_db (逻辑库名称)
------active_version (当前生效的版本)
------versions
--------0
----------data_sources (底层数据库信息)
----------rules (逻辑库的规则信息，例如分片规则，加密规则等)
------schemas (表、视图信息)
--------sharding_db
----------tables
------------t_order
------------t_single
----------views
----shardingsphere (内置元数据库)
------schemas
--------shardingsphere
----------tables
------------sharding_table_statics (分片统计信息表)
------------cluster_information （版本信息）
----performance_schema (模拟 mysql 数据库)
------schemas
--------performance_schema
----------tables
------------accounts
----information_schema (模拟 mysql 数据库)
------schemas
--------information_schema
----------tables
------------tables
------------schemata
------------columns
------------engines
------------routines
------------parameters
------------views
----mysql
----sys
--sys_data (内置元数据库的具体行信息)
----shardingsphere
------schemas
--------shardingsphere
----------tables
------------sharding_table_statistics
--------------79ff60bc40ab09395bed54cfecd08f94
--------------e832393209c9a4e7e117664c5ff8fc61
------------cluster_information
--------------d387c4f7de791e34d206f7dd59e24c1c

mysql> select * from shardingsphere.cluster_information;
+----------------+
| version        |
+----------------+
| 5.3.2-SNAPSHOT |
+----------------+
1 row in set (0.38 sec)

mysql> select * from shardingsphere.sharding_table_statistics where actual_table_name = "t_order_1"\G
**** 1. row ****
                  id: 2
 logic_database_name: sharding_db
    logic_table_name: t_order
actual_database_name: ds_0
   actual_table_name: t_order_1
           row_count: 0
                size: 16384
**** 2. row ****
                  id: 4
 logic_database_name: sharding_db
    logic_table_name: t_order
actual_database_name: ds_1
   actual_table_name: t_order_1
           row_count: 0
                size: 16384
2 rows in set (0.39 sec)

/**
 * ShardingSphere data collector.
 */
 @SingletonSPI
 public interface ShardingSphereDataCollector extends TypedSPI {
    
    /**
     * Collect.
     *
     * @param databaseName database name
     * @param table table
     * @param shardingSphereDatabases ShardingSphere databases
     * @return ShardingSphere table data
     * @throws SQLException sql exception
     */
    Optional<ShardingSphereTableData> collect(String databaseName, ShardingSphereTable table, Map<String, ShardingSphereDatabase> shardingSphereDatabases) throws SQLException;

/**
 * ShardingSphere schema data registry subscriber.
 */
 @SuppressWarnings("UnstableApiUsage")
 public final class ShardingSphereSchemaDataRegistrySubscriber {
    
    private final ShardingSphereDataPersistService persistService;
    
    private final GlobalLockPersistService lockPersistService;
    
    public ShardingSphereSchemaDataRegistrySubscriber(final ClusterPersistRepository repository, final GlobalLockPersistService globalLockPersistService, final EventBusContext eventBusContext){
        persistService = new ShardingSphereDataPersistService(repository);
        lockPersistService = globalLockPersistService;
        eventBusContext.register(this);
    }
    
    /**
     * Update when ShardingSphere schema data altered.
     *
     * @param event schema altered event
     */@Subscribepublic void update(final ShardingSphereSchemaDataAlteredEvent event){
        String databaseName = event.getDatabaseName();
        String schemaName = event.getSchemaName();
        GlobalLockDefinition lockDefinition = new GlobalLockDefinition("sys_data_" + event.getDatabaseName() + event.getSchemaName() + event.getTableName());
        if (lockPersistService.tryLock(lockDefinition, 10_000)) {
            try {
                persistService.getTableRowDataPersistService().persist(databaseName, schemaName, event.getTableName(), event.getAddedRows());
                persistService.getTableRowDataPersistService().persist(databaseName, schemaName, event.getTableName(), event.getUpdatedRows());
                persistService.getTableRowDataPersistService().delete(databaseName, schemaName, event.getTableName(), event.getDeletedRows());
            } finally {
                lockPersistService.unlock(lockDefinition);
            }
        }
    }
}

/**
 * ShardingSphere data changed watcher.
 */
 public final class ShardingSphereDataChangedWatcher implements GovernanceWatcher<GovernanceEvent> {
    
    @Override
    public Collection<String> getWatchingKeys(final String databaseName){
        return Collections.singleton(ShardingSphereDataNode.getShardingSphereDataNodePath());
    }
    
    @Override
    public Collection<Type> getWatchingTypes(){
        return Arrays.asList(Type.ADDED, Type.UPDATED, Type.DELETED);
    }
    
    @Override
    public Optional<GovernanceEvent> createGovernanceEvent(final DataChangedEvent event){
        if (isDatabaseChanged(event)) {
            return createDatabaseChangedEvent(event);
        }
        if (isSchemaChanged(event)) {
            return createSchemaChangedEvent(event);
        }
        if (isTableChanged(event)) {
            return createTableChangedEvent(event);
        }
        if (isTableRowDataChanged(event)) {
            return createRowDataChangedEvent(event);
        }
        return Optional.empty();
    }

    private Enumerable<Object[]> executeByShardingSphereData(final String databaseName, final String schemaName, final ShardingSphereTable table) {
        Optional<ShardingSphereTableData> tableData = Optional.ofNullable(data.getDatabaseData().get(databaseName)).map(optional -> optional.getSchemaData().get(schemaName))
                .map(ShardingSphereSchemaData::getTableData).map(shardingSphereData -> shardingSphereData.get(table.getName()));
        return tableData.map(this::createMemoryEnumerator).orElseGet(this::createEmptyEnumerable);
    }
    
    private Enumerable<Object[]> createMemoryEnumerator(final ShardingSphereTableData tableData) {
        return new AbstractEnumerable<Object[]>() {
            
            @Overridepublic Enumerator<Object[]> enumerator() {
                return new MemoryEnumerator<>(tableData.getRows());
            }
        };
    }

SELECT n.nspname as "Schema",
  c.relname as "Name",
  CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'i' THEN 'index' WHEN 'I' THEN 'global partition index' WHEN 'S' THEN 'sequence' WHEN 'L' THEN 'large sequence' WHEN 'f' THEN 'foreign table' WHEN 'm' THEN 'materialized view'  WHEN 'e' THEN 'stream' WHEN 'o' THEN 'contview' END as "Type",
  pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
  c.reloptions as "Storage"
FROM pg_catalog.pg_class c
     LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','v','m','S','L','f','e','o','')
      AND n.nspname <> 'pg_catalog'AND n.nspname <> 'db4ai'AND n.nspname <> 'information_schema'AND n.nspname !~ '^pg_toast'AND c.relname not like 'matviewmap\_%'AND c.relname not like 'mlog\_%'AND pg_catalog.pg_table_is_visible(c.oid)
ORDER BY 1,2;

pg_catalog.pg_class 
pg_catalog.pg_namespace

pg_catalog.pg_get_userbyid(c.relowner),
pg_catalog.pg_table_is_visible(c.oid)。

    /**
     * Create catalog reader.
     *
     * @param schemaName schema name
     * @param schema schema
     * @param relDataTypeFactory rel data type factory
     * @param connectionConfig connection config
     * @return calcite catalog reader
     */
     public static CalciteCatalogReader createCatalogReader(final String schemaName, final Schema schema, final RelDataTypeFactory relDataTypeFactory, final CalciteConnectionConfig connectionConfig){
        CalciteSchema rootSchema = CalciteSchema.createRootSchema(true);
        rootSchema.add(schemaName, schema);
        registryUserDefinedFunction(schemaName, rootSchema.plus());
        return new CalciteCatalogReader(rootSchema, Collections.singletonList(schemaName), relDataTypeFactory, connectionConfig);
    }
    
    private static void registryUserDefinedFunction(final String schemaName, final SchemaPlus schemaPlus){
        if (!"pg_catalog".equalsIgnoreCase(schemaName)) {
            return;
        }
        schemaPlus.add("pg_catalog.pg_table_is_visible", ScalarFunctionImpl.create(SQLFederationPlannerUtil.class, "pgTableIsVisible"));
        schemaPlus.add("pg_catalog.pg_get_userbyid", ScalarFunctionImpl.create(SQLFederationPlannerUtil.class, "pgGetUserById"));
    }

    /**
     * Mock pg_table_is_visible function.
     *
     * @param oid oid
     * @return true
     */
    @SuppressWarnings("unused")
    public static boolean pgTableIsVisible(final Long oid){
        return true;
    }
    
    /**
     * Mock pg_get_userbyid function.
     * 
     * @param oid oid
     * @return user name
     */
    @SuppressWarnings("unused")
    public static String pgGetUserById(final Long oid){
        return "mock user";
    }

sharding_db=> \d
                 List of relations
 Schema |   Name   |       Type        |   Owner
--------+----------+-------------------+-----------
 public | t_order  | table             | mock user
 public | t_single | table             | mock user
(2 rows)