Commit 40e08183 authored by Romain Reuillon's avatar Romain Reuillon

[Core] enh: implement section content for output formats

parent 9a6e9aa6
......@@ -53,9 +53,10 @@ package object csv {
header: Option[String] = None,
values: Seq[Any],
unrollArray: Boolean = false,
arrayOnRow: Boolean = false): Unit = {
arrayOnRow: Boolean = false,
margin: String = ""): Unit = {
header.foreach(h output.appendLine { h })
header.foreach(h output.appendLine { margin + h })
def csvLine(v: Seq[Any]): String = {
def format(v: Any): String =
......@@ -77,7 +78,7 @@ package object csv {
def unroll(v: Seq[Any]) = {
def writeLines(lists: Seq[List[Any]]): Unit = {
output.appendLine(csvLine(lists.map(_.head)))
output.appendLine(margin + csvLine(lists.map(_.head)))
val lastLine = lists.forall(_.tail.isEmpty)
if (!lastLine) {
......@@ -109,12 +110,12 @@ package object csv {
case v Seq(v)
}
output.appendLine(csvLine(arrayValues(v)))
output.appendLine(margin + csvLine(arrayValues(v)))
}
if (unrollArray) unroll(values)
else if (arrayOnRow) onRow(values)
else output.appendLine(csvLine(values))
else output.appendLine(margin + csvLine(values))
}
/**
......
package org.openmole.core.workflow.mole
import java.io.PrintStream
import org.openmole.core.csv
import org.openmole.core.workflow.mole
import org.openmole.core.workflow.tools.{ OptionalArgument, WritableOutput }
......@@ -31,25 +33,48 @@ object CSVHook {
object CSVOutputFormat {
implicit def format: OutputFormat[CSVOutputFormat, Any] = new OutputFormat[CSVOutputFormat, Any] {
override def write(format: CSVOutputFormat, output: WritableOutput, variables: Seq[Variable[_]], method: Any): FromContext[Unit] = FromContext { p
override def write(format: CSVOutputFormat, output: WritableOutput, content: OutputFormat.OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
def headerLine = format.header.map(_.from(context)) getOrElse csv.header(variables.map(_.prototype), variables.map(_.value), arrayOnRow = format.arrayOnRow)
def headerLine(variables: Seq[Variable[_]]) = format.header.map(_.from(context)) getOrElse csv.header(variables.map(_.prototype), variables.map(_.value), arrayOnRow = format.arrayOnRow)
def writeFile(f: File, variables: Seq[Variable[_]]) = {
val create = !format.append || f.isEmpty
val h = if (f.isEmpty) Some(headerLine(variables)) else None
if (create) f.atomicWithPrintStream { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
else f.withPrintStream(append = true, create = true) { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
}
output match {
case WritableOutput.FileValue(file)
val f = file.from(context)
val create = !format.append || f.isEmpty
def writeStream(ps: PrintStream, section: Option[String], variables: Seq[Variable[_]]) =
section match {
case None
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow)
case Some(section)
ps.println(section + ":")
val header = Some(headerLine(variables))
csv.writeVariablesToCSV(ps, header, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow, margin = " ")
}
val h = if (f.isEmpty) Some(headerLine) else None
import WritableOutput._
import OutputFormat._
if (create) f.atomicWithPrintStream { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
else f.withPrintStream(append = true, create = true) { ps csv.writeVariablesToCSV(ps, h, variables.map(_.value), unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow) }
(output, content) match {
case (FileValue(file), PlainContent(variables, name))
val f =
name match {
case None file.from(context)
case Some(name) file.from(context) / s"${name.from(context)}.csv"
}
case WritableOutput.StreamValue(ps, prelude)
prelude.foreach(ps.print)
val header = Some(headerLine)
csv.writeVariablesToCSV(ps, header, variables, unrollArray = format.unrollArray, arrayOnRow = format.arrayOnRow)
writeFile(f, variables)
case (FileValue(file), SectionContent(sections))
val directory = file.from(context)
for { section sections } writeFile(directory / s"${section.name.from(context)}.csv", section.variables)
case (StreamValue(ps), PlainContent(variables, name))
writeStream(ps, name.map(_.from(context)), variables)
case (StreamValue(ps), SectionContent(sections))
for { section sections } writeStream(ps, Some(section.name.from(context)), section.variables)
}
}
......@@ -57,8 +82,6 @@ object CSVHook {
import p._
format.header.option.toSeq.flatMap(_.validate(inputs))
}
override def extension: String = ".csv"
}
}
......
......@@ -8,10 +8,22 @@ import org.openmole.core.exception.UserBadDataError
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.builder._
object OutputFormat {
object OutputContent {
implicit def sectionToContent(s: Seq[OutputSection]) = SectionContent(s)
implicit def variablesToPlainContent(v: Seq[Variable[_]]) = PlainContent(v)
}
sealed trait OutputContent
case class SectionContent(sections: Seq[OutputSection]) extends OutputContent
case class PlainContent(variables: Seq[Variable[_]], name: Option[FromContext[String]] = None) extends OutputContent
case class OutputSection(name: FromContext[String], variables: Seq[Variable[_]])
}
trait OutputFormat[T, -M] {
def write(format: T, output: WritableOutput, variables: Seq[Variable[_]], method: M): FromContext[Unit]
def write(format: T, output: WritableOutput, content: OutputFormat.OutputContent, method: M): FromContext[Unit]
def validate(format: T): FromContextHook.ValidateParameters Seq[Throwable]
def extension: String
}
object FormattedFileHook {
......
......@@ -24,9 +24,13 @@ package mole {
def CSVHook = mole.CSVHook
def CSVOutputFormat = mole.CSVHook.CSVOutputFormat
type OutputFormat[T, D] = mole.OutputFormat[T, D]
val FromContextHook = mole.FromContextHook
type FromContextHook = mole.FromContextHook
type FromContextSource = mole.FromContextSource
val OutputFormat = mole.OutputFormat
type OutputFormat[T, D] = mole.OutputFormat[T, D]
type OutputContent = OutputFormat.OutputContent
}
}
......
......@@ -30,7 +30,7 @@ object WritableOutput {
type Display = PrintStream
case class FileValue(file: FromContext[File]) extends WritableOutput
case class StreamValue(stream: Display, prelude: Option[String] = None) extends WritableOutput
case class StreamValue(stream: PrintStream) extends WritableOutput
def file(writableOutput: WritableOutput) =
writableOutput match {
......
......@@ -2,32 +2,58 @@ package org.openmole.plugin.hook.json
import org.openmole.core.dsl._
import org.openmole.core.dsl.extension._
import org.openmole.core.workflow.mole.FromContextHook
import org.openmole.plugin.tool.json._
object JSONOutputFormat {
implicit def outputFormat = new OutputFormat[JSONOutputFormat, Any] {
override def write(format: JSONOutputFormat, output: WritableOutput, variables: Seq[Variable[_]], method: Any): FromContext[Unit] = FromContext { p
override def write(format: JSONOutputFormat, output: WritableOutput, content: OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
import org.json4s._
import org.json4s.jackson.JsonMethods._
implicit val formats = DefaultFormats
output match {
case WritableOutput.FileValue(file)
file.from(context).withPrintStream(append = false, create = true) { ps
import WritableOutput._
import OutputFormat._
import org.json4s._
import org.json4s.jackson.JsonMethods._
def sectionContent(sections: SectionContent) =
JObject(
sections.sections.map { section section.name.from(context) -> variablesToJValue(section.variables) }.toList
)
(output, content) match {
case (FileValue(file), PlainContent(variables, name))
val f =
name match {
case Some(n) file / s"${n.from(context)}.json"
case None file
}
f.from(context).withPrintStream(append = false, create = true) { ps
ps.print(compact(render(variablesToJValue(variables))))
}
case WritableOutput.StreamValue(ps, prelude)
prelude.foreach(ps.print)
ps.println(pretty(render(variablesToJValue(variables))))
case (FileValue(file), sections: SectionContent)
file.from(context).withPrintStream(append = false, create = true) { ps
ps.print(compact(render(sectionContent(sections))))
}
case (StreamValue(ps), PlainContent(variables, name))
name match {
case Some(f)
ps.println(s"${f.from(context)}:")
ps.println(pretty(render(variablesToJValue(variables))).split("\n").map(" " + _).mkString("\n"))
case None ps.println(pretty(render(variablesToJValue(variables))))
}
case (StreamValue(ps), sections: SectionContent)
ps.println(pretty(render(sectionContent(sections))))
}
}
override def validate(format: JSONOutputFormat): FromContextHook.ValidateParameters Seq[Throwable] = { p Seq() }
override def extension = ".json"
}
}
......
......@@ -36,31 +36,16 @@ object SavePopulationHook {
Hook("SavePopulationHook") { p
import p._
def saveFile(dir: FromContext[File]) =
def fileName =
(frequency, last) match {
case (_, true) Some(dir / ExpandedString("population" + outputFormat.extension))
case (None, _) Some(dir / ExpandedString("population${" + t.generationPrototype.name + "}" + outputFormat.extension))
case (Some(f), _) if context(t.generationPrototype) % f == 0
Some(dir / ExpandedString("population${" + t.generationPrototype.name + "}" + outputFormat.extension))
case (_, true) Some(ExpandedString("population"))
case (None, _) Some(ExpandedString("population${" + t.generationPrototype.name + "}"))
case (Some(f), _) if context(t.generationPrototype) % f == 0 Some(ExpandedString("population${" + t.generationPrototype.name + "}"))
case _ None
}
output match {
case WritableOutput.FileValue(dir)
saveFile(dir) match {
case Some(outputFile) outputFormat.write(format, outputFile.from(context), resultVariables(t).from(context), evolutionData(t)).from(context)
case None
}
case o
val save =
(frequency, last) match {
case (_, true) true
case (Some(f), _) context(t.generationPrototype) % f == 0
case _ false
}
if (save) outputFormat.write(format, o, resultVariables(t).from(context), evolutionData(t)).from(context)
}
val section = OutputFormat.PlainContent(resultVariables(t).from(context), fileName)
outputFormat.write(format, output, section, evolutionData(t)).from(context)
context
} validate { p outputFormat.validate(format)(p) } set (inputs += (t.populationPrototype, t.statePrototype))
......
......@@ -99,7 +99,7 @@ package object evolution {
implicit def workflowIntegration = WorkflowIntegration[DSLContainer[EvolutionWorkflow]](_.data)
implicit class EvolutionMethodContainer(dsl: DSLContainer[EvolutionWorkflow]) extends DSLContainerHook(dsl) {
def hook[F](output: WritableOutput, frequency: OptionalArgument[Long] = None, last: Boolean = false, format: F = CSVOutputFormat())(implicit outputFormat: OutputFormat[F, SavePopulationHook.EvolutionData]): DSLContainer[EvolutionWorkflow] = {
def hook[F](output: WritableOutput, frequency: OptionalArgument[Long] = None, last: Boolean = false, format: F = CSVOutputFormat(unrollArray = true))(implicit outputFormat: OutputFormat[F, SavePopulationHook.EvolutionData]): DSLContainer[EvolutionWorkflow] = {
implicit val defScope = dsl.scope
dsl.hook(SavePopulationHook(dsl, output, frequency = frequency, last = last, format = format))
}
......
......@@ -12,16 +12,17 @@ object MorrisHook {
val inputs = ScalarOrSequenceOfDouble.prototypes(dsl.data.inputs)
output match {
case FileValue(dirFC)
Sensitivity.writeResults(format, dsl.data, FileValue(dirFC / s"mu${outputFormat.extension}"), inputs, dsl.data.outputs, Morris.mu(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, FileValue(dirFC / s"muStar${outputFormat.extension}"), inputs, dsl.data.outputs, Morris.muStar(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, FileValue(dirFC / s"sigma${outputFormat.extension}"), inputs, dsl.data.outputs, Morris.sigma(_, _)).from(context)
case StreamValue(ps, prelude)
Sensitivity.writeResults(format, dsl.data, StreamValue(ps, Some(prelude.getOrElse("") + "mu\n")), inputs, dsl.data.outputs, Morris.mu(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, StreamValue(ps, Some("muStar\n")), inputs, dsl.data.outputs, Morris.muStar(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, StreamValue(ps, Some("sigma\n")), inputs, dsl.data.outputs, Morris.sigma(_, _)).from(context)
}
import OutputFormat._
def sections =
Seq(
OutputSection("mu", Sensitivity.variableResults(inputs, dsl.data.outputs, Morris.mu(_, _)).from(context)),
OutputSection("muStar", Sensitivity.variableResults(inputs, dsl.data.outputs, Morris.muStar(_, _)).from(context)),
OutputSection("sigma", Sensitivity.variableResults(inputs, dsl.data.outputs, Morris.sigma(_, _)).from(context))
)
outputFormat.write(format, output, sections, dsl.data).from(context)
context
}
......
......@@ -12,14 +12,15 @@ object SaltelliHook {
val inputs = ScalarOrSequenceOfDouble.prototypes(dsl.data.inputs)
output match {
case FileValue(dirFC)
Sensitivity.writeResults(format, dsl.data, FileValue(dirFC / s"firstOrderIndices${outputFormat.extension}"), inputs, dsl.data.outputs, Saltelli.firstOrder(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, FileValue(dirFC / s"totalOrderIndices${outputFormat.extension}"), inputs, dsl.data.outputs, Saltelli.totalOrder(_, _)).from(context)
case StreamValue(ps, prelude)
Sensitivity.writeResults(format, dsl.data, StreamValue(ps, Some(prelude.getOrElse("") + "first order\n")), inputs, dsl.data.outputs, Saltelli.firstOrder(_, _)).from(context)
Sensitivity.writeResults(format, dsl.data, StreamValue(ps, Some("total order\n")), inputs, dsl.data.outputs, Saltelli.totalOrder(_, _)).from(context)
}
import OutputFormat._
def sections =
Seq(
OutputSection("firstOrderIndices", Sensitivity.variableResults(inputs, dsl.data.outputs, Saltelli.firstOrder(_, _)).from(context)),
OutputSection("totalOrderIndices", Sensitivity.variableResults(inputs, dsl.data.outputs, Saltelli.totalOrder(_, _)).from(context))
)
outputFormat.write(format, output, sections, dsl.data).from(context)
context
}
......
......@@ -53,7 +53,7 @@ package object sensitivity {
}
def writeResults[F, D](format: F, method: D, output: WritableOutput, inputs: Seq[Val[_]], outputs: Seq[Val[_]], coefficient: (Val[_], Val[_]) Val[_])(implicit outputFormat: OutputFormat[F, D]) = FromContext { p
def variableResults[F, D](inputs: Seq[Val[_]], outputs: Seq[Val[_]], coefficient: (Val[_], Val[_]) Val[_])(implicit outputFormat: OutputFormat[F, D]) = FromContext { p
import p._
def results = outputs.map { o
......@@ -62,9 +62,7 @@ package object sensitivity {
}
def allVals = Seq(Val[String]("output")) ++ inputs
val data = (results.transpose zip allVals).map { case (value, v) => Variable.unsecure(v.array, value) }
outputFormat.write(format, output, data, method = method).from(context)
(results.transpose zip allVals).map { case (value, v) => Variable.unsecure(v.array, value) }
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment