Package org.apache.sysds.runtime.controlprogram.federated

Class FederationUtils

java.lang.Object

org.apache.sysds.runtime.controlprogram.federated.FederationUtils

public class FederationUtils
extends Object

Constructor Summary

Constructors

Constructor	Description
FederationUtils()

Method Summary

Modifier and Type	Method	Description
static MatrixBlock	aggAdd(Future<FederatedResponse>[] ffr)
static boolean	aggBooleanScalar(Future<FederatedResponse>[] tmp)
static MatrixBlock	aggMatrix(AggregateUnaryOperator aop, Future<FederatedResponse>[] ffr, Future<FederatedResponse>[] meanFfr, FederationMap map)
static MatrixBlock	aggMatrix(AggregateUnaryOperator aop, Future<FederatedResponse>[] ffr, FederationMap map)
static MatrixBlock	aggMean(Future<FederatedResponse>[] ffr, FederationMap map)
static MatrixBlock	aggMinMax(Future<FederatedResponse>[] ffr, boolean isMin, boolean isScalar, Optional<FTypes.FType> fedType)
static MatrixBlock	aggMinMaxIndex(Future<FederatedResponse>[] ffr, boolean isMin, FederationMap map)
static MatrixBlock	aggProd(Future<FederatedResponse>[] ffr, FederationMap fedMap, AggregateUnaryOperator aop)
static MatrixBlock	aggregateResponses(List<org.apache.commons.lang3.tuple.Pair<FederatedRange,Future<FederatedResponse>>> readResponses)	Aggregate partially aggregated data from federated workers by adding values with the same index in different federated locations.
static ScalarObject	aggScalar(AggregateUnaryOperator aop, Future<FederatedResponse>[] ffr)
static ScalarObject	aggScalar(AggregateUnaryOperator aop, Future<FederatedResponse>[] ffr, Future<FederatedResponse>[] meanFfr, FederationMap map)
static ScalarObject	aggScalar(AggregateUnaryOperator aop, Future<FederatedResponse>[] ffr, FederationMap map)
static MatrixBlock	aggVar(Future<FederatedResponse>[] ffr, Future<FederatedResponse>[] meanFfr, FederationMap map, boolean isRowAggregate, boolean isScalar)
static MatrixBlock	bind(Future<FederatedResponse>[] ffr, boolean cbind)
static MatrixBlock	bindResponses(List<org.apache.commons.lang3.tuple.Pair<FederatedRange,Future<FederatedResponse>>> readResponses, long[] dims)	Bind data from federated workers based on non-overlapping federated ranges.
static FederatedRequest[]	callInstruction(String[] inst, CPOperand varOldOut, long outputId, CPOperand[] varOldIn, long[] varNewIn, Types.ExecType type)
static FederatedRequest[]	callInstruction(String[] inst, CPOperand varOldOut, CPOperand[] varOldIn, long[] varNewIn)
static FederatedRequest	callInstruction(String inst, CPOperand varOldOut, long outputId, CPOperand[] varOldIn, long[] varNewIn, Types.ExecType type, boolean rmFedOutputFlag)
static FederatedRequest	callInstruction(String inst, CPOperand varOldOut, CPOperand[] varOldIn, long[] varNewIn)
static FederatedRequest	callInstruction(String inst, CPOperand varOldOut, CPOperand[] varOldIn, long[] varNewIn, boolean rmFedOutFlag)
static void	checkFedMapType(MatrixObject mo)
static Optional<io.netty.channel.ChannelInboundHandlerAdapter>	compressionDecoder()
static Optional<io.netty.channel.ChannelOutboundHandlerAdapter>	compressionEncoder()
static Optional<org.apache.commons.lang3.tuple.ImmutablePair<io.netty.channel.ChannelInboundHandlerAdapter,io.netty.channel.ChannelOutboundHandlerAdapter>>	compressionStrategy()
static io.netty.handler.codec.serialization.ObjectDecoder	decoder()
static FederationMap	federateLocalData(CacheableData<?> data)
static long	getNextFedDataID()
static MatrixBlock[]	getResults(Future<FederatedResponse>[] ffr)
static void	resetFedDataID()
static long	sumNonZeros(Future<FederatedResponse>[] responses)
static void	waitFor(List<Future<FederatedResponse>> responses)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FederationUtils

public FederationUtils()

Method Detail

resetFedDataID

public static void resetFedDataID()

getNextFedDataID

public static long getNextFedDataID()

checkFedMapType

public static void checkFedMapType(MatrixObject mo)

callInstruction

public static FederatedRequest callInstruction(String inst,
                                               CPOperand varOldOut,
                                               CPOperand[] varOldIn,
                                               long[] varNewIn,
                                               boolean rmFedOutFlag)

callInstruction

public static FederatedRequest callInstruction(String inst,
                                               CPOperand varOldOut,
                                               CPOperand[] varOldIn,
                                               long[] varNewIn)

callInstruction

public static FederatedRequest[] callInstruction(String[] inst,
                                                 CPOperand varOldOut,
                                                 CPOperand[] varOldIn,
                                                 long[] varNewIn)

callInstruction

public static FederatedRequest[] callInstruction(String[] inst,
                                                 CPOperand varOldOut,
                                                 long outputId,
                                                 CPOperand[] varOldIn,
                                                 long[] varNewIn,
                                                 Types.ExecType type)

callInstruction

public static FederatedRequest callInstruction(String inst,
                                               CPOperand varOldOut,
                                               long outputId,
                                               CPOperand[] varOldIn,
                                               long[] varNewIn,
                                               Types.ExecType type,
                                               boolean rmFedOutputFlag)

aggAdd

public static MatrixBlock aggAdd(Future<FederatedResponse>[] ffr)

aggMean

public static MatrixBlock aggMean(Future<FederatedResponse>[] ffr,
                                  FederationMap map)

getResults

public static MatrixBlock[] getResults(Future<FederatedResponse>[] ffr)

bind

public static MatrixBlock bind(Future<FederatedResponse>[] ffr,
                               boolean cbind)

aggMinMax

public static MatrixBlock aggMinMax(Future<FederatedResponse>[] ffr,
                                    boolean isMin,
                                    boolean isScalar,
                                    Optional<FTypes.FType> fedType)

aggProd

public static MatrixBlock aggProd(Future<FederatedResponse>[] ffr,
                                  FederationMap fedMap,
                                  AggregateUnaryOperator aop)

aggMinMaxIndex

public static MatrixBlock aggMinMaxIndex(Future<FederatedResponse>[] ffr,
                                         boolean isMin,
                                         FederationMap map)

aggVar

public static MatrixBlock aggVar(Future<FederatedResponse>[] ffr,
                                 Future<FederatedResponse>[] meanFfr,
                                 FederationMap map,
                                 boolean isRowAggregate,
                                 boolean isScalar)

aggScalar

public static ScalarObject aggScalar(AggregateUnaryOperator aop,
                                     Future<FederatedResponse>[] ffr,
                                     Future<FederatedResponse>[] meanFfr,
                                     FederationMap map)

aggMatrix

public static MatrixBlock aggMatrix(AggregateUnaryOperator aop,
                                    Future<FederatedResponse>[] ffr,
                                    Future<FederatedResponse>[] meanFfr,
                                    FederationMap map)

waitFor

public static void waitFor(List<Future<FederatedResponse>> responses)

aggScalar

public static ScalarObject aggScalar(AggregateUnaryOperator aop,
                                     Future<FederatedResponse>[] ffr)

aggScalar

public static ScalarObject aggScalar(AggregateUnaryOperator aop,
                                     Future<FederatedResponse>[] ffr,
                                     FederationMap map)

aggBooleanScalar

public static boolean aggBooleanScalar(Future<FederatedResponse>[] tmp)

aggMatrix

public static MatrixBlock aggMatrix(AggregateUnaryOperator aop,
                                    Future<FederatedResponse>[] ffr,
                                    FederationMap map)

federateLocalData

public static FederationMap federateLocalData(CacheableData<?> data)

bindResponses

public static MatrixBlock bindResponses(List<org.apache.commons.lang3.tuple.Pair<FederatedRange,Future<FederatedResponse>>> readResponses,
                                        long[] dims)
                                 throws Exception

Bind data from federated workers based on non-overlapping federated ranges.

Parameters:

readResponses - responses from federated workers containing the federated ranges and data

dims - dimensions of output MatrixBlock

Returns:

MatrixBlock of consolidated data

Throws:

Exception - in case of problems with getting data from responses

aggregateResponses

public static MatrixBlock aggregateResponses(List<org.apache.commons.lang3.tuple.Pair<FederatedRange,Future<FederatedResponse>>> readResponses)

Aggregate partially aggregated data from federated workers by adding values with the same index in different federated locations.

Parameters:

readResponses - responses from federated workers containing the federated data

Returns:

MatrixBlock of consolidated, aggregated data

decoder

public static io.netty.handler.codec.serialization.ObjectDecoder decoder()

compressionEncoder

public static Optional<io.netty.channel.ChannelOutboundHandlerAdapter> compressionEncoder()

compressionDecoder

public static Optional<io.netty.channel.ChannelInboundHandlerAdapter> compressionDecoder()

compressionStrategy

public static Optional<org.apache.commons.lang3.tuple.ImmutablePair<io.netty.channel.ChannelInboundHandlerAdapter,io.netty.channel.ChannelOutboundHandlerAdapter>> compressionStrategy()

sumNonZeros

public static long sumNonZeros(Future<FederatedResponse>[] responses)