Package org.apache.sysds.runtime.transform.encode

Class MultiColumnEncoder

java.lang.Object

org.apache.sysds.runtime.transform.encode.MultiColumnEncoder

All Implemented Interfaces:

Externalizable, Serializable, Encoder

public class MultiColumnEncoder
extends Object
implements Encoder

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field	Description
static boolean	APPLY_ENCODER_SEPARATE_STAGES
static boolean	MULTI_THREADED_STAGES

Constructor Summary

Constructors

Constructor	Description
MultiColumnEncoder()
MultiColumnEncoder(List<ColumnEncoderComposite> columnEncoders)

Method Summary

Modifier and Type	Method	Description
<T extends LegacyEncoder> void	addReplaceLegacyEncoder(T encoder)
void	allocateMetaData(FrameBlock meta)	Pre-allocate a FrameBlock for metadata collection.
MatrixBlock	apply(CacheBlock<?> in)
MatrixBlock	apply(CacheBlock<?> in, int k)
MatrixBlock	apply(CacheBlock<?> in, MatrixBlock out, int outputCol)	Apply the generated metadata to the FrameBlock and saved the result in out.
MatrixBlock	apply(CacheBlock<?> in, MatrixBlock out, int outputCol, int k)
void	applyColumnOffset()
<T extends ColumnEncoder,E> void	applyToAll(Class<T> type, Consumer<? super T> function)
<T extends ColumnEncoder,E> void	applyToAll(Consumer<? super ColumnEncoderComposite> function)
void	build(CacheBlock<?> in)	Build the transform meta data for the given block input.
void	build(CacheBlock<?> in, int k)
void	build(CacheBlock<?> in, int k, Map<Integer,double[]> equiHeightBinMaxs)
void	buildPartial(FrameBlock in)	Partial build of internal data structures (e.g., in distributed spark operations).
<T extends ColumnEncoder> boolean	containsEncoderForID(int colID, Class<T> type)
MatrixBlock	encode(CacheBlock<?> in)
MatrixBlock	encode(CacheBlock<?> in, boolean compressedOut)
MatrixBlock	encode(CacheBlock<?> in, int k)
MatrixBlock	encode(CacheBlock<?> in, int k, boolean compressedOut)
MatrixBlock	getColMapping(FrameBlock meta)	Obtain the column mapping of encoded frames based on the passed meta data frame.
<T extends ColumnEncoder> T	getColumnEncoder(int colID, Class<T> type)
List<ColumnEncoderComposite>	getColumnEncoders()
<T extends ColumnEncoder> List<T>	getColumnEncoders(Class<T> type)
List<ColumnEncoderComposite>	getCompositeEncodersForID(int colID)
List<Class<? extends ColumnEncoder>>	getEncoderTypes()
List<Class<? extends ColumnEncoder>>	getEncoderTypes(int colID)
int	getEstNNzRow()
<T extends ColumnEncoder,E> List<E>	getFromAll(Class<T> type, Function<? super T,? extends E> mapper)
<T extends ColumnEncoder> double[]	getFromAllDoubleArray(Class<T> type, Function<? super T,? extends Double> mapper)
<T extends ColumnEncoder> int[]	getFromAllIntArray(Class<T> type, Function<? super T,? extends Integer> mapper)
<T extends LegacyEncoder> T	getLegacyEncoder(Class<T> type)
FrameBlock	getMetaData(FrameBlock meta)	Construct a frame block out of the transform meta data.
FrameBlock	getMetaData(FrameBlock meta, int k)
int	getNumExtraCols(IndexRange ixRange)
int	getNumOutCols()
<T extends LegacyEncoder> boolean	hasLegacyEncoder()
<T extends LegacyEncoder> boolean	hasLegacyEncoder(Class<T> type)
void	initEmbeddings(MatrixBlock embeddings)
void	initMetaData(FrameBlock meta)	Sets up the required meta data for a subsequent call to apply.
boolean	isCompressedTransformEncode(CacheBlock<?> in, boolean enabled)
void	legacyBuild(FrameBlock in)
void	mergeAt(Encoder other, int columnOffset, int row)
void	mergeReplace(MultiColumnEncoder multiEncoder)
void	prepareBuildPartial()	Allocates internal data structures for partial build.
void	readExternal(ObjectInput in)
MultiColumnEncoder	subRangeEncoder(IndexRange ixRange)
<T extends ColumnEncoder> MultiColumnEncoder	subRangeEncoder(IndexRange ixRange, Class<T> type)
String	toString()
void	updateAllDCEncoders()
void	updateIndexRanges(long[] beginDims, long[] endDims, int offset)	Update index-ranges to after encoding.
void	writeExternal(ObjectOutput out)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

MULTI_THREADED_STAGES

public static boolean MULTI_THREADED_STAGES

APPLY_ENCODER_SEPARATE_STAGES

public static boolean APPLY_ENCODER_SEPARATE_STAGES

Constructor Detail

MultiColumnEncoder

public MultiColumnEncoder(List<ColumnEncoderComposite> columnEncoders)

MultiColumnEncoder

public MultiColumnEncoder()

Method Detail

encode

public MatrixBlock encode(CacheBlock<?> in)

encode

public MatrixBlock encode(CacheBlock<?> in,
                          int k)

encode

public MatrixBlock encode(CacheBlock<?> in,
                          boolean compressedOut)

encode

public MatrixBlock encode(CacheBlock<?> in,
                          int k,
                          boolean compressedOut)

build

public void build(CacheBlock<?> in)

Description copied from interface: Encoder

Build the transform meta data for the given block input. This call modifies and keeps meta data as encoder state.

Specified by:

build in interface Encoder

Parameters:

in - input frame block

build

public void build(CacheBlock<?> in,
                  int k)

build

public void build(CacheBlock<?> in,
                  int k,
                  Map<Integer,double[]> equiHeightBinMaxs)

legacyBuild

public void legacyBuild(FrameBlock in)

apply

public MatrixBlock apply(CacheBlock<?> in)

apply

public MatrixBlock apply(CacheBlock<?> in,
                         int k)

updateAllDCEncoders

public void updateAllDCEncoders()

apply

public MatrixBlock apply(CacheBlock<?> in,
                         MatrixBlock out,
                         int outputCol)

Description copied from interface: Encoder

Apply the generated metadata to the FrameBlock and saved the result in out.

Specified by:

apply in interface Encoder

Parameters:

in - input frame block

out - output matrix block

outputCol - is a offset in the output matrix. column in FrameBlock + outputCol = column in out

Returns:

output matrix block

apply

public MatrixBlock apply(CacheBlock<?> in,
                         MatrixBlock out,
                         int outputCol,
                         int k)

allocateMetaData

public void allocateMetaData(FrameBlock meta)

Description copied from interface: Encoder

Pre-allocate a FrameBlock for metadata collection.

Specified by:

allocateMetaData in interface Encoder

Parameters:

meta - frame block

getMetaData

public FrameBlock getMetaData(FrameBlock meta)

Description copied from interface: Encoder

Construct a frame block out of the transform meta data.

Specified by:

getMetaData in interface Encoder

Parameters:

meta - output frame block

Returns:

output frame block?

getMetaData

public FrameBlock getMetaData(FrameBlock meta,
                              int k)

initMetaData

public void initMetaData(FrameBlock meta)

Description copied from interface: Encoder

Sets up the required meta data for a subsequent call to apply.

Specified by:

initMetaData in interface Encoder

Parameters:

meta - frame block

initEmbeddings

public void initEmbeddings(MatrixBlock embeddings)

prepareBuildPartial

public void prepareBuildPartial()

Description copied from interface: Encoder

Allocates internal data structures for partial build.

Specified by:

prepareBuildPartial in interface Encoder

buildPartial

public void buildPartial(FrameBlock in)

Description copied from interface: Encoder

Partial build of internal data structures (e.g., in distributed spark operations).

Specified by:

buildPartial in interface Encoder

Parameters:

in - input frame block

getColMapping

public MatrixBlock getColMapping(FrameBlock meta)

Obtain the column mapping of encoded frames based on the passed meta data frame.

Parameters:

meta - meta data frame block

Returns:

matrix with column mapping (one row per attribute)

updateIndexRanges

public void updateIndexRanges(long[] beginDims,
                              long[] endDims,
                              int offset)

Description copied from interface: Encoder

Update index-ranges to after encoding. Note that only Dummycoding changes the ranges.

Specified by:

updateIndexRanges in interface Encoder

Parameters:

beginDims - begin dimensions of range

endDims - end dimensions of range

offset - is applied to begin and endDims

writeExternal

public void writeExternal(ObjectOutput out)
                   throws IOException

Specified by:

writeExternal in interface Externalizable

Throws:

IOException

readExternal

public void readExternal(ObjectInput in)
                  throws IOException,
                         ClassNotFoundException

Specified by:

readExternal in interface Externalizable

Throws:

IOException

ClassNotFoundException

getColumnEncoders

public <T extends ColumnEncoder> List<T> getColumnEncoders(Class<T> type)

getColumnEncoder

public <T extends ColumnEncoder> T getColumnEncoder(int colID,
                                                    Class<T> type)

getFromAll

public <T extends ColumnEncoder,E> List<E> getFromAll(Class<T> type,
                                                            Function<? super T,? extends E> mapper)

getFromAllIntArray

public <T extends ColumnEncoder> int[] getFromAllIntArray(Class<T> type,
                                                          Function<? super T,? extends Integer> mapper)

getFromAllDoubleArray

public <T extends ColumnEncoder> double[] getFromAllDoubleArray(Class<T> type,
                                                                Function<? super T,? extends Double> mapper)

getColumnEncoders

public List<ColumnEncoderComposite> getColumnEncoders()

getCompositeEncodersForID

public List<ColumnEncoderComposite> getCompositeEncodersForID(int colID)

getEncoderTypes

public List<Class<? extends ColumnEncoder>> getEncoderTypes(int colID)

getEncoderTypes

public List<Class<? extends ColumnEncoder>> getEncoderTypes()

getEstNNzRow

public int getEstNNzRow()

getNumOutCols

public int getNumOutCols()

getNumExtraCols

public int getNumExtraCols(IndexRange ixRange)

containsEncoderForID

public <T extends ColumnEncoder> boolean containsEncoderForID(int colID,
                                                              Class<T> type)

applyToAll

public <T extends ColumnEncoder,E> void applyToAll(Class<T> type,
                                                         Consumer<? super T> function)

applyToAll

public <T extends ColumnEncoder,E> void applyToAll(Consumer<? super ColumnEncoderComposite> function)

subRangeEncoder

public MultiColumnEncoder subRangeEncoder(IndexRange ixRange)

subRangeEncoder

public <T extends ColumnEncoder> MultiColumnEncoder subRangeEncoder(IndexRange ixRange,
                                                                    Class<T> type)

mergeReplace

public void mergeReplace(MultiColumnEncoder multiEncoder)

mergeAt

public void mergeAt(Encoder other,
                    int columnOffset,
                    int row)

addReplaceLegacyEncoder

public <T extends LegacyEncoder> void addReplaceLegacyEncoder(T encoder)

hasLegacyEncoder

public <T extends LegacyEncoder> boolean hasLegacyEncoder()

isCompressedTransformEncode

public boolean isCompressedTransformEncode(CacheBlock<?> in,
                                           boolean enabled)

hasLegacyEncoder

public <T extends LegacyEncoder> boolean hasLegacyEncoder(Class<T> type)

getLegacyEncoder

public <T extends LegacyEncoder> T getLegacyEncoder(Class<T> type)

applyColumnOffset

public void applyColumnOffset()

toString

public String toString()

Overrides:

toString in class Object