public class MetaStoreUtils
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
static java.lang.String |
DATABASE_WAREHOUSE_SUFFIX |
static java.lang.String |
DEFAULT_DATABASE_COMMENT |
static java.lang.String |
DEFAULT_DATABASE_NAME |
protected static org.apache.commons.logging.Log |
LOG |
Constructor and Description |
---|
MetaStoreUtils() |
Modifier and Type | Method and Description |
---|---|
static boolean |
compareFieldColumns(java.util.List<FieldSchema> schema1,
java.util.List<FieldSchema> schema2) |
static Table |
createColumnsetSchema(java.lang.String name,
java.util.List<java.lang.String> columns,
java.util.List<java.lang.String> partCols,
Configuration conf) |
static void |
deleteWHDirectory(Path path,
Configuration conf,
boolean use_trash) |
static int |
findFreePort()
Finds a free port on the machine.
|
static java.lang.Class<?> |
getClass(java.lang.String rawStoreClassName) |
static java.lang.String |
getColumnNamesFromFieldSchema(java.util.List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnNames.
|
static java.lang.String |
getColumnTypesFromFieldSchema(java.util.List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnTypes.
|
static java.lang.String |
getDDLFromFieldSchema(java.lang.String structName,
java.util.List<FieldSchema> fieldSchemas)
Convert FieldSchemas to Thrift DDL.
|
static Deserializer |
getDeserializer(Configuration conf,
Partition part,
Table table)
getDeserializer
Get the Deserializer for a partition.
|
static Deserializer |
getDeserializer(Configuration conf,
java.util.Properties schema)
getDeserializer
Get the Deserializer for a table given its name and properties.
|
static Deserializer |
getDeserializer(Configuration conf,
Table table)
getDeserializer
Get the Deserializer for a table.
|
static FieldSchema |
getFieldSchemaFromTypeInfo(java.lang.String fieldName,
TypeInfo typeInfo)
Convert TypeInfo to FieldSchema.
|
static java.util.List<FieldSchema> |
getFieldsFromDeserializer(java.lang.String tableName,
Deserializer deserializer) |
static java.lang.String |
getFullDDLFromFieldSchema(java.lang.String structName,
java.util.List<FieldSchema> fieldSchemas)
Convert FieldSchemas to Thrift DDL + column names and column types
|
static java.lang.String |
getIndexTableName(java.lang.String dbName,
java.lang.String baseTblName,
java.lang.String indexName) |
static java.lang.String |
getListType(java.lang.String t) |
static java.lang.String |
getMapType(java.lang.String k,
java.lang.String v) |
static java.util.Map<java.lang.String,java.lang.String> |
getMetaStoreSaslProperties(HiveConf conf)
Read and return the meta store Sasl configuration.
|
static Path |
getOriginalLocation(Partition part) |
static java.util.Properties |
getPartitionMetadata(Partition partition,
Table table) |
static java.util.Properties |
getPartSchemaFromTableSchema(StorageDescriptor sd,
StorageDescriptor tblsd,
java.util.Map<java.lang.String,java.lang.String> parameters,
java.lang.String databaseName,
java.lang.String tableName,
java.util.List<FieldSchema> partitionKeys,
java.util.Properties tblSchema)
Get partition level schema from table level schema.
|
static java.util.List<java.lang.String> |
getPvals(java.util.List<FieldSchema> partCols,
java.util.Map<java.lang.String,java.lang.String> partSpec)
Given a list of partition columns and a partial mapping from
some partition columns to values the function returns the values
for the column.
|
static java.util.Properties |
getSchema(Partition part,
Table table) |
static java.util.Properties |
getSchema(StorageDescriptor sd,
StorageDescriptor tblsd,
java.util.Map<java.lang.String,java.lang.String> parameters,
java.lang.String databaseName,
java.lang.String tableName,
java.util.List<FieldSchema> partitionKeys) |
static java.util.Properties |
getTableMetadata(Table table) |
static boolean |
isArchived(Partition part) |
static boolean |
isExternalTable(Table table)
Determines whether a table is an external table.
|
static boolean |
isIndexTable(Table table) |
static boolean |
isNonNativeTable(Table table) |
static void |
makeDir(Path path,
HiveConf hiveConf) |
static java.lang.String |
makeFilterStringFromMap(java.util.Map<java.lang.String,java.lang.String> m)
Given a map of partition column names to values, this creates a filter
string that can be used to call the *byFilter methods
|
static <T> T |
newInstance(java.lang.Class<T> theClass,
java.lang.Class<?>[] parameterTypes,
java.lang.Object[] initargs)
Create an object of the given class.
|
static boolean |
partitionNameHasValidCharacters(java.util.List<java.lang.String> partVals,
java.util.regex.Pattern partitionValidationPattern) |
static void |
printStackTrace(java.lang.Exception e)
printStackTrace
Helper function to print an exception stack trace to the log and not stderr
|
static boolean |
pvalMatches(java.util.List<java.lang.String> partial,
java.util.List<java.lang.String> full)
Returns true if partial has the same values as full for all values that
aren't empty in partial.
|
static void |
recursiveDelete(java.io.File f)
recursiveDelete
just recursively deletes a dir - you'd think Java would have something to
do this??
|
static void |
setSerdeParam(SerDeInfo sdi,
java.util.Properties schema,
java.lang.String param) |
static void |
startMetaStore(int port,
HadoopThriftAuthBridge bridge) |
static java.lang.String |
typeToThriftType(java.lang.String type)
Convert type to ThriftType.
|
static boolean |
validateColumnType(java.lang.String type)
validate column type
if it is predefined, yes.
|
static boolean |
validateName(java.lang.String name)
validateName
Checks the name conforms to our standars which are: "[a-zA-z_0-9]+".
|
static void |
validatePartitionNameCharacters(java.util.List<java.lang.String> partVals,
java.util.regex.Pattern partitionValidationPattern) |
static java.lang.String |
validateSkewedColNames(java.util.List<java.lang.String> cols) |
static java.lang.String |
validateSkewedColNamesSubsetCol(java.util.List<java.lang.String> skewedColNames,
java.util.List<FieldSchema> cols) |
static java.lang.String |
validateTblColumns(java.util.List<FieldSchema> cols) |
protected static final org.apache.commons.logging.Log LOG
public static final java.lang.String DEFAULT_DATABASE_NAME
public static final java.lang.String DEFAULT_DATABASE_COMMENT
public static final java.lang.String DATABASE_WAREHOUSE_SUFFIX
public static void printStackTrace(java.lang.Exception e)
e
- the exceptionpublic static Table createColumnsetSchema(java.lang.String name, java.util.List<java.lang.String> columns, java.util.List<java.lang.String> partCols, Configuration conf) throws MetaException
MetaException
public static void recursiveDelete(java.io.File f) throws java.io.IOException
f
- - the file/dir to deletejava.io.IOException
- propogate f.delete() exceptionspublic static Deserializer getDeserializer(Configuration conf, java.util.Properties schema) throws MetaException
conf
- hadoop configschema
- the properties to use to instantiate the deserializerMetaException
- if any problems instantiating the Deserializer
todo - this should move somewhere into serde.jarpublic static Deserializer getDeserializer(Configuration conf, Table table) throws MetaException
conf
- - hadoop configtable
- the tableMetaException
- if any problems instantiating the Deserializer
todo - this should move somewhere into serde.jarpublic static Deserializer getDeserializer(Configuration conf, Partition part, Table table) throws MetaException
conf
- - hadoop configpart
- the partitiontable
- the tableMetaException
- if any problems instantiating the Deserializerpublic static void deleteWHDirectory(Path path, Configuration conf, boolean use_trash) throws MetaException
MetaException
public static java.util.List<java.lang.String> getPvals(java.util.List<FieldSchema> partCols, java.util.Map<java.lang.String,java.lang.String> partSpec)
partCols
- the list of table partition columnspartSpec
- the partial mapping from partition column to valuespublic static boolean validateName(java.lang.String name)
name
- the name to validateMetaException
- if it doesn't match the pattern.public static java.lang.String validateTblColumns(java.util.List<FieldSchema> cols)
public static boolean validateColumnType(java.lang.String type)
name
- public static java.lang.String validateSkewedColNames(java.util.List<java.lang.String> cols)
public static java.lang.String validateSkewedColNamesSubsetCol(java.util.List<java.lang.String> skewedColNames, java.util.List<FieldSchema> cols)
public static java.lang.String getListType(java.lang.String t)
public static java.lang.String getMapType(java.lang.String k, java.lang.String v)
public static void setSerdeParam(SerDeInfo sdi, java.util.Properties schema, java.lang.String param)
public static java.lang.String typeToThriftType(java.lang.String type)
public static java.lang.String getFullDDLFromFieldSchema(java.lang.String structName, java.util.List<FieldSchema> fieldSchemas)
structName
- The name of the tablefieldSchemas
- List of fields along with their schemaspublic static java.lang.String getDDLFromFieldSchema(java.lang.String structName, java.util.List<FieldSchema> fieldSchemas)
public static java.util.Properties getTableMetadata(Table table)
public static java.util.Properties getPartitionMetadata(Partition partition, Table table)
public static java.util.Properties getPartSchemaFromTableSchema(StorageDescriptor sd, StorageDescriptor tblsd, java.util.Map<java.lang.String,java.lang.String> parameters, java.lang.String databaseName, java.lang.String tableName, java.util.List<FieldSchema> partitionKeys, java.util.Properties tblSchema)
sd
- The Partition level Storage Descriptor.tblsd
- The Table level Storage Descriptor.parameters
- partition level parametersdatabaseName
- DB nametableName
- table namepartitionKeys
- partition columnstblSchema
- The table level schema from which this partition should be copied.public static java.util.Properties getSchema(StorageDescriptor sd, StorageDescriptor tblsd, java.util.Map<java.lang.String,java.lang.String> parameters, java.lang.String databaseName, java.lang.String tableName, java.util.List<FieldSchema> partitionKeys)
public static java.lang.String getColumnNamesFromFieldSchema(java.util.List<FieldSchema> fieldSchemas)
public static java.lang.String getColumnTypesFromFieldSchema(java.util.List<FieldSchema> fieldSchemas)
public static void makeDir(Path path, HiveConf hiveConf) throws MetaException
MetaException
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge) throws java.lang.Exception
java.lang.Exception
public static int findFreePort() throws java.io.IOException
java.io.IOException
public static java.util.List<FieldSchema> getFieldsFromDeserializer(java.lang.String tableName, Deserializer deserializer) throws SerDeException, MetaException
tableName
- deserializer
- SerDeException
MetaException
public static FieldSchema getFieldSchemaFromTypeInfo(java.lang.String fieldName, TypeInfo typeInfo)
public static boolean isExternalTable(Table table)
table
- table of interestpublic static boolean isArchived(Partition part)
public static Path getOriginalLocation(Partition part)
public static boolean isNonNativeTable(Table table)
public static boolean pvalMatches(java.util.List<java.lang.String> partial, java.util.List<java.lang.String> full)
public static java.lang.String getIndexTableName(java.lang.String dbName, java.lang.String baseTblName, java.lang.String indexName)
public static boolean isIndexTable(Table table)
public static java.lang.String makeFilterStringFromMap(java.util.Map<java.lang.String,java.lang.String> m)
m
- public static java.lang.Class<?> getClass(java.lang.String rawStoreClassName) throws MetaException
MetaException
public static <T> T newInstance(java.lang.Class<T> theClass, java.lang.Class<?>[] parameterTypes, java.lang.Object[] initargs)
theClass
- parameterTypes
- an array of parameterTypes for the constructorinitargs
- the list of arguments for the constructorpublic static void validatePartitionNameCharacters(java.util.List<java.lang.String> partVals, java.util.regex.Pattern partitionValidationPattern) throws MetaException
MetaException
public static boolean partitionNameHasValidCharacters(java.util.List<java.lang.String> partVals, java.util.regex.Pattern partitionValidationPattern)
public static boolean compareFieldColumns(java.util.List<FieldSchema> schema1, java.util.List<FieldSchema> schema2)
schema1:
- The first schema to be comparedschema2:
- The second schema to be comparedpublic static java.util.Map<java.lang.String,java.lang.String> getMetaStoreSaslProperties(HiveConf conf)
conf
- Copyright © 2012 The Apache Software Foundation