Commit 9fa387e2 authored by Romain Reuillon's avatar Romain Reuillon

[Core] enh: create format and hook packages in workflow

parent 40e08183
......@@ -18,7 +18,7 @@
package org.openmole.modules
import org.openmole.core.module._
import org.openmole.core.workflow.mole.CSVHook
import org.openmole.core.workflow.hook.CSVHook
import org.openmole.plugin.environment.condor.CondorEnvironment
import org.openmole.plugin.environment.egi._
import org.openmole.plugin.environment.oar.OAREnvironment
......
......@@ -13,9 +13,9 @@ package object extension {
type ScalarOrSequenceOfDouble[T] = org.openmole.core.workflow.tools.ScalarOrSequenceOfDouble[T]
def ScalarOrSequenceOfDouble = org.openmole.core.workflow.tools.ScalarOrSequenceOfDouble
type Hook = org.openmole.core.workflow.mole.FromContextHook
def Hook = org.openmole.core.workflow.mole.Hook
def FormattedFileHook = org.openmole.core.workflow.mole.FormattedFileHook
type Hook = org.openmole.core.workflow.hook.FromContextHook
def Hook = org.openmole.core.workflow.hook.Hook
def FormattedFileHook = org.openmole.core.workflow.hook.FormattedFileHook
type Source = org.openmole.core.workflow.mole.Source
type FromContextSource = org.openmole.core.workflow.mole.FromContextSource
......@@ -41,8 +41,12 @@ package object extension {
type Context = org.openmole.core.context.Context
def Context = org.openmole.core.context.Context
type WritableOutput = org.openmole.core.workflow.tools.WritableOutput
val WritableOutput = org.openmole.core.workflow.tools.WritableOutput
type WritableOutput = org.openmole.core.workflow.format.WritableOutput
val WritableOutput = org.openmole.core.workflow.format.WritableOutput
val OutputFormat = org.openmole.core.workflow.format.OutputFormat
type OutputFormat[T, D] = org.openmole.core.workflow.format.OutputFormat[T, D]
type OutputContent = OutputFormat.OutputContent
def ExpandedString = org.openmole.core.expansion.ExpandedString
......
......@@ -2,7 +2,7 @@ package org.openmole.core.workflow
import org.openmole.core.pluginmanager.PluginInfo
import org.openmole.core.preference.ConfigurationInfo
import org.openmole.core.workflow.mole.CSVHook.CSVOutputFormat
import org.openmole.core.workflow.format.CSVOutputFormat
import org.osgi.framework.{ BundleActivator, BundleContext }
class Activator extends BundleActivator {
......@@ -54,8 +54,6 @@ class Activator extends BundleActivator {
TaskKeyWord(objectName(ClosureTask)),
TaskKeyWord(objectName(ToArrayTask)),
TaskKeyWord(objectName(MoleTask)),
TaskKeyWord(objectName(FromContextTask)),
HookKeyWord(objectName(FromContextHook)),
OtherKeyWord(objectName(CSVOutputFormat))
)
}
......
......@@ -31,12 +31,13 @@ package composition {
import org.openmole.core.outputmanager.OutputManager
import org.openmole.core.workflow.builder.DefinitionScope
import org.openmole.core.workflow.execution.{ EnvironmentProvider, LocalEnvironmentProvider }
import org.openmole.core.workflow.mole.CSVHook.CSVOutputFormat
import org.openmole.core.workflow.mole.{ CSVHook, OutputFormat, FormattedFileHook, Grouping, Hook, MasterCapsule, Mole, MoleCapsule, MoleExecution, MoleExecutionContext, MoleServices, Source }
import org.openmole.core.workflow.format.{ OutputFormat, WritableOutput, CSVOutputFormat }
import org.openmole.core.workflow.hook.{ FormattedFileHook, Hook }
import org.openmole.core.workflow.mole.{ Grouping, MasterCapsule, Mole, MoleCapsule, MoleExecution, MoleExecutionContext, MoleServices, Source }
import org.openmole.core.workflow.sampling.Sampling
import org.openmole.core.workflow.task.{ EmptyTask, ExplorationTask, Task }
import org.openmole.core.workflow.tools.{ OptionalArgument, WritableOutput }
import org.openmole.core.workflow.tools.WritableOutput.Display
import org.openmole.core.workflow.tools.OptionalArgument
import org.openmole.core.workflow.format.WritableOutput.Display
import org.openmole.core.workflow.transition._
import org.openmole.core.workflow.validation.TypeUtil
import shapeless.{ ::, HList }
......
package org.openmole.core.workflow.format
import java.io.PrintStream
import org.openmole.core.context.{ Val, Variable }
import org.openmole.core.csv
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.tools.OptionalArgument
object CSVOutputFormat {
implicit def format: OutputFormat[CSVOutputFormat, Any] = new OutputFormat[CSVOutputFormat, Any] {
override def write(format: CSVOutputFormat, output: WritableOutput, content: OutputFormat.OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
def headerLine(variables: Seq[Variable[_]]) = format.header.map(_.from(context)) getOrElse csv.header(variables.map(_.prototype), variables.map(_.value), arrayOnRow = format.arrayOnRow)
def writeFile(f: File, variables: Seq[Variable[_]]) = {
val create = !format.append || f.isEmpty
val h = if (f.isEmpty) Some(headerLine(variables)) else None
if (create) f.atomicWithPrintStream { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
else f.withPrintStream(append = true, create = true) { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
}
def writeStream(ps: PrintStream, section: Option[String], variables: Seq[Variable[_]]) =
section match {
case None
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow)
case Some(section)
ps.println(section + ":")
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow, margin = " ")
}
import OutputFormat._
import WritableOutput._
(output, content) match {
case (FileValue(file), PlainContent(variables, name))
val f =
name match {
case None file.from(context)
case Some(name) file.from(context) / s"${name.from(context)}.csv"
}
writeFile(f, variables)
case (FileValue(file), SectionContent(sections))
val directory = file.from(context)
for { section sections } writeFile(directory / s"${section.name.from(context)}.csv", section.variables)
case (StreamValue(ps), PlainContent(variables, name))
writeStream(ps, name.map(_.from(context)), variables)
case (StreamValue(ps), SectionContent(sections))
for { section sections } writeStream(ps, Some(section.name.from(context)), section.variables)
}
}
override def validate(format: CSVOutputFormat) = { p
import p._
format.header.option.toSeq.flatMap(_.validate(inputs))
}
}
}
case class CSVOutputFormat(
header: OptionalArgument[FromContext[String]] = None,
unrollArray: Boolean = false,
arrayOnRow: Boolean = false,
append: Boolean = false)
\ No newline at end of file
package org.openmole.core.workflow.format
import org.openmole.core.context._
import org.openmole.core.expansion._
import org.openmole.core.workflow.hook.FromContextHook
object OutputFormat {
object OutputContent {
implicit def sectionToContent(s: Seq[OutputSection]) = SectionContent(s)
implicit def variablesToPlainContent(v: Seq[Variable[_]]) = PlainContent(v)
}
sealed trait OutputContent
case class SectionContent(sections: Seq[OutputSection]) extends OutputContent
case class PlainContent(variables: Seq[Variable[_]], name: Option[FromContext[String]] = None) extends OutputContent
case class OutputSection(name: FromContext[String], variables: Seq[Variable[_]])
}
trait OutputFormat[T, -M] {
def write(format: T, output: WritableOutput, content: OutputFormat.OutputContent, method: M): FromContext[Unit]
def validate(format: T): FromContextHook.ValidateParameters Seq[Throwable]
}
\ No newline at end of file
package org.openmole.core.workflow
package format {
import org.openmole.tool.outputredirection._
trait FormatPackage {
type Display = WritableOutput.Display
def display(implicit outputRedirection: OutputRedirection): Display = outputRedirection.output
def CSVOutputFormat = format.CSVOutputFormat
}
}
package object format {
}
package org.openmole.core.workflow.hook
import java.io.PrintStream
import org.openmole.core.context.{ Val, Variable }
import org.openmole.core.csv
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder.DefinitionScope
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.format.{ CSVOutputFormat, WritableOutput }
import org.openmole.core.workflow.tools.OptionalArgument
object CSVHook {
def apply(output: WritableOutput, values: Val[_]*)(implicit name: sourcecode.Name, definitionScope: DefinitionScope): FromContextHook =
apply(output, values.toVector)
def apply(
output: WritableOutput,
values: Seq[Val[_]] = Vector.empty,
exclude: Seq[Val[_]] = Vector.empty,
header: OptionalArgument[FromContext[String]] = None,
unrollArray: Boolean = false,
overwrite: Boolean = false)(implicit name: sourcecode.Name, definitionScope: DefinitionScope): FromContextHook =
FormattedFileHook(
format = CSVOutputFormat(header = header, unrollArray = unrollArray, append = !overwrite),
output = output,
values = values,
exclude = exclude,
name = Some("CSVHook")
)
}
package org.openmole.core.workflow.mole
package org.openmole.core.workflow.hook
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.mole
import org.openmole.core.workflow.tools.WritableOutput
import org.openmole.core.context._
import org.openmole.core.exception.UserBadDataError
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
object OutputFormat {
object OutputContent {
implicit def sectionToContent(s: Seq[OutputSection]) = SectionContent(s)
implicit def variablesToPlainContent(v: Seq[Variable[_]]) = PlainContent(v)
}
sealed trait OutputContent
case class SectionContent(sections: Seq[OutputSection]) extends OutputContent
case class PlainContent(variables: Seq[Variable[_]], name: Option[FromContext[String]] = None) extends OutputContent
case class OutputSection(name: FromContext[String], variables: Seq[Variable[_]])
}
trait OutputFormat[T, -M] {
def write(format: T, output: WritableOutput, content: OutputFormat.OutputContent, method: M): FromContext[Unit]
def validate(format: T): FromContextHook.ValidateParameters Seq[Throwable]
}
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.format._
object FormattedFileHook {
......@@ -34,7 +15,7 @@ object FormattedFileHook {
values: Seq[Val[_]] = Vector.empty,
exclude: Seq[Val[_]] = Vector.empty,
method: M = None,
name: Option[String] = None)(implicit valName: sourcecode.Name, definitionScope: DefinitionScope, fileFormat: OutputFormat[T, M]): mole.FromContextHook =
name: Option[String] = None)(implicit valName: sourcecode.Name, definitionScope: DefinitionScope, fileFormat: OutputFormat[T, M]): FromContextHook =
Hook(name getOrElse "FileFormatHook") { parameters
import parameters._
......
......@@ -15,7 +15,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.openmole.core.workflow.mole
package org.openmole.core.workflow.hook
import org.openmole.core.context.Context
import org.openmole.core.expansion.FromContext
......
package org.openmole.core.workflow
package hook {
trait HookPackage {
def CSVHook = hook.CSVHook
val FromContextHook = hook.FromContextHook
type FromContextHook = hook.FromContextHook
}
}
package object hook {
def Hook = FromContextHook
}
package org.openmole.core.workflow.mole
import java.io.PrintStream
import org.openmole.core.csv
import org.openmole.core.workflow.mole
import org.openmole.core.workflow.tools.{ OptionalArgument, WritableOutput }
import org.openmole.core.context._
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.dsl._
object CSVHook {
def apply(output: WritableOutput, values: Val[_]*)(implicit name: sourcecode.Name, definitionScope: DefinitionScope): mole.FromContextHook =
apply(output, values.toVector)
def apply(
output: WritableOutput,
values: Seq[Val[_]] = Vector.empty,
exclude: Seq[Val[_]] = Vector.empty,
header: OptionalArgument[FromContext[String]] = None,
unrollArray: Boolean = false,
overwrite: Boolean = false)(implicit name: sourcecode.Name, definitionScope: DefinitionScope): mole.FromContextHook =
FormattedFileHook(
format = CSVOutputFormat(header = header, unrollArray = unrollArray, append = !overwrite),
output = output,
values = values,
exclude = exclude,
name = Some("CSVHook")
)
object CSVOutputFormat {
implicit def format: OutputFormat[CSVOutputFormat, Any] = new OutputFormat[CSVOutputFormat, Any] {
override def write(format: CSVOutputFormat, output: WritableOutput, content: OutputFormat.OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
def headerLine(variables: Seq[Variable[_]]) = format.header.map(_.from(context)) getOrElse csv.header(variables.map(_.prototype), variables.map(_.value), arrayOnRow = format.arrayOnRow)
def writeFile(f: File, variables: Seq[Variable[_]]) = {
val create = !format.append || f.isEmpty
val h = if (f.isEmpty) Some(headerLine(variables)) else None
if (create) f.atomicWithPrintStream { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
else f.withPrintStream(append = true, create = true) { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
}
def writeStream(ps: PrintStream, section: Option[String], variables: Seq[Variable[_]]) =
section match {
case None
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow)
case Some(section)
ps.println(section + ":")
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow, margin = " ")
}
import WritableOutput._
import OutputFormat._
(output, content) match {
case (FileValue(file), PlainContent(variables, name))
val f =
name match {
case None file.from(context)
case Some(name) file.from(context) / s"${name.from(context)}.csv"
}
writeFile(f, variables)
case (FileValue(file), SectionContent(sections))
val directory = file.from(context)
for { section sections } writeFile(directory / s"${section.name.from(context)}.csv", section.variables)
case (StreamValue(ps), PlainContent(variables, name))
writeStream(ps, name.map(_.from(context)), variables)
case (StreamValue(ps), SectionContent(sections))
for { section sections } writeStream(ps, Some(section.name.from(context)), section.variables)
}
}
override def validate(format: CSVOutputFormat) = { p
import p._
format.header.option.toSeq.flatMap(_.validate(inputs))
}
}
}
case class CSVOutputFormat(
header: OptionalArgument[FromContext[String]] = None,
unrollArray: Boolean = false,
arrayOnRow: Boolean = false,
append: Boolean = false)
}
......@@ -27,6 +27,7 @@ import org.openmole.core.exception.{ InternalProcessingError, UserBadDataError }
import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.execution._
import org.openmole.core.workflow.hook.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.job.State._
import org.openmole.core.workflow.job._
import org.openmole.core.workflow.mole
......
......@@ -17,26 +17,19 @@
package org.openmole.core.workflow
import org.openmole.core.workflow.hook.Hook
import scala.language.implicitConversions
package mole {
trait MolePackage {
def CSVHook = mole.CSVHook
def CSVOutputFormat = mole.CSVHook.CSVOutputFormat
val FromContextHook = mole.FromContextHook
type FromContextHook = mole.FromContextHook
trait MolePackage {
type FromContextSource = mole.FromContextSource
val OutputFormat = mole.OutputFormat
type OutputFormat[T, D] = mole.OutputFormat[T, D]
type OutputContent = OutputFormat.OutputContent
}
}
package object mole {
def Hook = FromContextHook
def Source = FromContextSource
case class Hooks(map: Map[MoleCapsule, Traversable[Hook]])
......
......@@ -26,6 +26,8 @@ package workflow {
import org.openmole.core.workflow.builder.BuilderPackage
import org.openmole.core.workflow.mole.MolePackage
import org.openmole.core.workflow.composition.CompositionPackage
import org.openmole.core.workflow.format.FormatPackage
import org.openmole.core.workflow.hook.HookPackage
import org.openmole.core.workflow.sampling.SamplingPackage
import org.openmole.core.workflow.task.TaskPackage
import org.openmole.core.workflow.tools.ToolsPackage
......@@ -42,6 +44,8 @@ package workflow {
with ContextPackage
with ExpansionPackage
with KeyWordPackage
with FormatPackage
with HookPackage
object dsl extends ExportedPackage
......
......@@ -20,7 +20,7 @@ import monocle.macros.Lenses
import org.openmole.core.context.Context
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.mole.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.hook.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.test.Stubs._
object TestHook {
......
......@@ -46,9 +46,6 @@ package tools {
def apply[S](s: S)(implicit expandable: Expandable[S, T]) = expandable.expand(s)
}
type Display = tools.WritableOutput.Display
def display(implicit outputRedirection: OutputRedirection): Display = outputRedirection.output
}
}
......@@ -18,7 +18,8 @@
package org.openmole.core.workflow.validation
import org.openmole.core.context.{ Val, ValType }
import org.openmole.core.workflow.mole.{ Hook, MoleCapsule, Source }
import org.openmole.core.workflow.hook.Hook
import org.openmole.core.workflow.mole.{ MoleCapsule, Source }
import org.openmole.core.workflow.transition.TransitionSlot
import org.openmole.core.workflow.validation.TypeUtil.InvalidType
......
......@@ -20,7 +20,8 @@ package org.openmole.core.workflow.validation
import org.openmole.core.context.Val
import org.openmole.core.fileservice.FileService
import org.openmole.core.tools.io.Prettifier
import org.openmole.core.workflow.mole.{ Hook, _ }
import org.openmole.core.workflow.hook.Hook
import org.openmole.core.workflow.mole._
import org.openmole.core.workflow.task._
import org.openmole.core.workflow.transition._
import org.openmole.core.workspace.TmpDirectory
......
......@@ -19,7 +19,7 @@ package org.openmole.plugin.hook.file
import org.openmole.core.pluginmanager._
import org.openmole.core.preference.ConfigurationInfo
import org.openmole.core.workflow.mole.CSVHook
import org.openmole.core.workflow.hook.CSVHook
import org.osgi.framework.BundleContext
class Activator extends PluginInfoActivator {
......
......@@ -19,13 +19,9 @@ package org.openmole.plugin.hook.file
import java.io.File
import monocle.macros.Lenses
import org.openmole.core.context.Context
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
import org.openmole.core.dsl._
import org.openmole.core.workflow.mole.{ MoleExecutionContext, _ }
import org.openmole.core.workflow.validation._
import org.openmole.core.dsl.extension._
import org.openmole.tool.stream._
object AppendToFileHook {
......
......@@ -25,6 +25,7 @@ import org.openmole.core.context.{ Context, Val, Variable }
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.hook.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.mole.{ MoleExecutionContext, _ }
import org.openmole.core.workflow.validation._
import org.openmole.plugin.hook.file.CopyFileHook._
......
......@@ -5,6 +5,7 @@ import org.openmole.core.context.Context
import org.openmole.core.dsl._
import org.openmole.core.expansion._
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.hook.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.mole._
import org.openmole.core.workflow.validation._
......
......@@ -25,6 +25,7 @@ import org.openmole.core.exception._
import org.openmole.core.expansion._
import org.openmole.core.serializer._
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.hook.{ Hook, HookExecutionContext }
import org.openmole.core.workflow.mole._