public class RemoteParForUtils extends Object
| Constructor and Description |
|---|
RemoteParForUtils() |
| Modifier and Type | Method and Description |
|---|---|
static void |
cleanupWorkingDirectories()
Cleanup all temporary files created by this SystemDS process.
|
static void |
cleanupWorkingDirectoriesOnShutdown()
Cleanup all temporary files created by this SystemDS process,
on shutdown via exit or interrupt.
|
static void |
exportLineageItems(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
Lineage lineage)
Export lineage for remote Spark parfor workers.
|
static ArrayList<String> |
exportResultVariables(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars)
For remote Spark parfor workers.
|
static void |
exportResultVariables(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
HashMap<String,String> rvarFnames,
org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable> out)
For remote MR parfor workers.
|
static void |
exportResultVariables(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable> out) |
static Lineage[] |
getLineages(LocalVariableMap[] results) |
static LocalVariableMap[] |
getResults(List<scala.Tuple2<Long,String>> out,
org.apache.commons.logging.Log LOG) |
static void |
incrementParForMRCounters(org.apache.hadoop.mapred.Reporter reporter,
long deltaTasks,
long deltaIterations) |
static void |
setupBufferPool(long workerID)
Init and register-cleanup of buffer pool
|
public static void incrementParForMRCounters(org.apache.hadoop.mapred.Reporter reporter,
long deltaTasks,
long deltaIterations)
public static void exportResultVariables(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable> out)
throws IOException
IOExceptionpublic static void exportResultVariables(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
HashMap<String,String> rvarFnames,
org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable> out)
throws IOException
workerID - worker idvars - local variable mapresultVars - list of result variablesrvarFnames - ?out - output collectorsIOException - if IOException occurspublic static ArrayList<String> exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ParForStatementBlock.ResultVar> resultVars) throws IOException
workerID - worker idvars - local variable mapresultVars - list of result variablesIOException - if IOException occurspublic static void exportLineageItems(long workerID,
LocalVariableMap vars,
ArrayList<ParForStatementBlock.ResultVar> resultVars,
Lineage lineage)
throws IOException
workerID - worker idvars - local variable mapresultVars - list of result variableslineage - lineage objectIOException - if IOException occurspublic static void cleanupWorkingDirectories()
public static void cleanupWorkingDirectoriesOnShutdown()
public static Lineage[] getLineages(LocalVariableMap[] results)
public static LocalVariableMap[] getResults(List<scala.Tuple2<Long,String>> out, org.apache.commons.logging.Log LOG)
public static void setupBufferPool(long workerID)
throws IOException
workerID - worker idIOException - exceptionCopyright © 2020 The Apache Software Foundation. All rights reserved.