Commit b12b6d42 authored by Romain Reuillon's avatar Romain Reuillon

[Core + Plugin] enh: alpha version of OMR output format

parent 20dc1ca4
......@@ -84,7 +84,6 @@ lazy val xstream = OsgiProject(
"!com.ctc.wstx.stax.*",
"!net.sf.cglib.*",
"!nu.xom.*",
"!org.codehaus.jettison.*",
"!org.dom4j.*",
"!org.jdom.*",
"!org.jdom2.*",
......@@ -97,8 +96,8 @@ lazy val xstream = OsgiProject(
"!javax.*",
"*"),
privatePackages = Seq("!scala.*", "META-INF.services.*", "*")) settings(
libraryDependencies ++= Seq("com.thoughtworks.xstream" % "xstream" % "1.4.11.1", "net.sf.kxml" % "kxml2" % "2.3.0"),
version := "1.4.11.1") settings(settings: _*)
libraryDependencies ++= Seq("com.thoughtworks.xstream" % "xstream" % "1.4.12", "net.sf.kxml" % "kxml2" % "2.3.0", "org.codehaus.jettison" % "jettison" % "1.4.1"),
version := "1.4.12") settings(settings: _*)
lazy val scalaLang = OsgiProject(
dir,
......
......@@ -159,7 +159,8 @@ class Runtime {
outputRedirection = outputRedirection,
loggerService = loggerService,
cache = KeyValueCache(),
lockRepository = LockRepository[LockKey]())
lockRepository = LockRepository[LockKey](),
serializerService = serializerService)
for (toProcess allMoleJobs) environment.submit(toProcess, taskExecutionContext)
saver.waitAllFinished
......
......@@ -454,8 +454,8 @@ lazy val modifierHook = OsgiProject(pluginDir, "org.openmole.plugin.hook.modifie
lazy val jsonHook = OsgiProject(pluginDir, "org.openmole.plugin.hook.json", imports = Seq("*")) dependsOn(openmoleDSL,json, replication % "test") settings (
libraryDependencies += Libraries.scalatest) settings (pluginSettings: _*)
lazy val omrHook = OsgiProject(pluginDir, "org.openmole.plugin.hook.omr", imports = Seq("*")) dependsOn(openmoleDSL,json, replication % "test") settings (
libraryDependencies += Libraries.scalatest) settings (pluginSettings: _*)
lazy val omrHook = OsgiProject(pluginDir, "org.openmole.plugin.hook.omr", imports = Seq("*")) dependsOn(openmoleDSL, jsonHook, replication % "test") settings (
libraryDependencies += Libraries.scalatest, libraryDependencies += Libraries.circe) settings (pluginSettings: _*)
/* Method */
......@@ -463,8 +463,11 @@ lazy val omrHook = OsgiProject(pluginDir, "org.openmole.plugin.hook.omr", import
def allMethod = Seq(evolution, directSampling, sensitivity, abc)
lazy val evolution = OsgiProject(pluginDir, "org.openmole.plugin.method.evolution", imports = Seq("*")) dependsOn(
openmoleDSL, toolsTask, pattern, collectionDomain % "test", boundsDomain % "test"
) settings(libraryDependencies += Libraries.mgo, libraryDependencies += Libraries.shapeless) settings (pluginSettings: _*)
openmoleDSL, toolsTask, pattern, collectionDomain % "test", boundsDomain % "test", omrHook % "test"
) settings(
libraryDependencies += Libraries.mgo,
libraryDependencies += Libraries.shapeless,
libraryDependencies += Libraries.circe) settings (pluginSettings: _*)
lazy val abc = OsgiProject(pluginDir, "org.openmole.plugin.method.abc", imports = Seq("*")) dependsOn(openmoleDSL, toolsTask, pattern, boundsDomain % "test") settings(
libraryDependencies += Libraries.mgo, libraryDependencies += Libraries.shapeless) settings (pluginSettings: _*)
......
......@@ -47,6 +47,7 @@ package object extension {
val OutputFormat = org.openmole.core.workflow.format.OutputFormat
type OutputFormat[T, D] = org.openmole.core.workflow.format.OutputFormat[T, D]
type OutputContent = OutputFormat.OutputContent
type HookExecutionContext = org.openmole.core.workflow.hook.HookExecutionContext
def ExpandedString = org.openmole.core.expansion.ExpandedString
......
......@@ -30,6 +30,7 @@ import org.openmole.core.serializer.converter._
import java.util.concurrent.locks.{ ReadWriteLock, ReentrantReadWriteLock }
import com.thoughtworks.xstream.converters.Converter
import com.thoughtworks.xstream.io.json._
import com.thoughtworks.xstream.mapper.Mapper
import com.thoughtworks.xstream.security._
import org.openmole.core.workspace.{ TmpDirectory, Workspace }
......@@ -50,13 +51,16 @@ object SerializerService {
*/
class SerializerService { service
private[serializer] def buildXStream() = {
private[serializer] def buildXStream(json: Boolean = false) = {
val lookup = new DefaultConverterLookup()
val driver =
if (json) new JsonHierarchicalStreamDriver() else new BinaryStreamDriver()
val xs =
new XStream(
null,
new BinaryStreamDriver(),
driver,
new ClassLoaderReference(this.getClass.getClassLoader),
null: Mapper,
lookup,
......@@ -71,7 +75,7 @@ class SerializerService { service ⇒
private val content = "content.xml"
private def fileSerialisation() = buildXStream
private def fileSerialisation() = buildXStream()
private def pluginAndFileListing() = new Serialiser(service) with PluginAndFilesListing
private def deserializerWithFileInjection() = new Serialiser(service) with FileInjection
......@@ -131,9 +135,9 @@ class SerializerService { service ⇒
def serialize(obj: Any, os: OutputStream) = buildXStream().toXML(obj, os)
def serialize(obj: Any, file: File): Unit = {
def serialize(obj: Any, file: File, json: Boolean = false): Unit = {
val os = file.bufferedOutputStream()
try serialize(obj, os)
try buildXStream(json = json).toXML(obj, os)
finally os.close
}
......
......@@ -21,6 +21,6 @@ import com.thoughtworks.xstream.XStream
import org.openmole.core.serializer.SerializerService
class Serialiser(serialiserService: SerializerService) {
val xStream: XStream = serialiserService.buildXStream
val xStream: XStream = serialiserService.buildXStream()
def clean = {}
}
......@@ -6,12 +6,13 @@ import org.openmole.core.context.{ Val, Variable }
import org.openmole.core.csv
import org.openmole.core.expansion.FromContext
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.hook.HookExecutionContext
import org.openmole.core.workflow.tools.OptionalArgument
object CSVOutputFormat {
implicit def format: OutputFormat[CSVOutputFormat, Any] = new OutputFormat[CSVOutputFormat, Any] {
override def write(format: CSVOutputFormat, output: WritableOutput, content: OutputFormat.OutputContent, method: Any): FromContext[Unit] = FromContext { p
override def write(executionContext: HookExecutionContext)(format: CSVOutputFormat, output: WritableOutput, content: OutputFormat.OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
def headerLine(variables: Seq[Variable[_]]) = format.header.map(_.from(context)) getOrElse csv.header(variables.map(_.prototype), variables.map(_.value), arrayOnRow = format.arrayOnRow)
......
......@@ -2,7 +2,7 @@ package org.openmole.core.workflow.format
import org.openmole.core.context._
import org.openmole.core.expansion._
import org.openmole.core.workflow.hook.FromContextHook
import org.openmole.core.workflow.hook.{ FromContextHook, HookExecutionContext }
object OutputFormat {
object OutputContent {
......@@ -18,6 +18,6 @@ object OutputFormat {
}
trait OutputFormat[T, -M] {
def write(format: T, output: WritableOutput, content: OutputFormat.OutputContent, method: M): FromContext[Unit]
def write(executionContext: HookExecutionContext)(format: T, output: WritableOutput, content: OutputFormat.OutputContent, method: M): FromContext[Unit]
def validate(format: T): FromContextHook.ValidateParameters Seq[Throwable]
}
\ No newline at end of file
......@@ -23,7 +23,8 @@ import org.openmole.core.expansion.FromContext
object WritableOutput {
implicit def fromFile(file: File) = Store(file)
implicit def fromString(s: String): Store = fromFile(new File(s))
implicit def fromFile(file: File): Store = Store(file)
implicit def fromFileContext(file: FromContext[File]) = Store(file)
implicit def fromPrintStream(ps: PrintStream) = Display(ps)
......
......@@ -24,7 +24,7 @@ object FormattedFileHook {
val ps = { if (values.isEmpty) context.values.map { _.prototype }.toVector else values }.filter { v !excludeSet.contains(v.name) }
val variables = ps.map(p context.variable(p).getOrElse(throw new UserBadDataError(s"Variable $p not found in hook $this")))
fileFormat.write(format, output, variables, method).from(context)
fileFormat.write(executionContext)(format, output, variables, method).from(context)
context
} validate { p
......
......@@ -5,6 +5,7 @@ import org.openmole.core.context._
import org.openmole.core.expansion._
import org.openmole.core.fileservice.FileService
import org.openmole.core.preference.Preference
import org.openmole.core.serializer.SerializerService
import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.validation
......@@ -21,16 +22,19 @@ object FromContextHook {
implicit def isInfo = InfoBuilder(FromContextHook.info)
case class Parameters(
context: Context,
cache: KeyValueCache,
implicit val preference: Preference,
implicit val threadProvider: ThreadProvider,
implicit val fileService: FileService,
implicit val workspace: Workspace,
implicit val outputRedirection: OutputRedirection,
implicit val loggerService: LoggerService,
implicit val random: RandomProvider,
implicit val newFile: TmpDirectory)
context: Context,
cache: KeyValueCache,
executionContext: HookExecutionContext) {
implicit def preference: Preference = executionContext.preference
implicit def threadProvider: ThreadProvider = executionContext.threadProvider
implicit def fileService: FileService = executionContext.fileService
implicit def workspace: Workspace = executionContext.workspace
implicit def outputRedirection: OutputRedirection = executionContext.outputRedirection
implicit def loggerService: LoggerService = executionContext.loggerService
implicit def random: RandomProvider = executionContext.random
implicit def newFile: TmpDirectory = executionContext.newFile
implicit def serializerService: SerializerService = executionContext.serializerService
}
case class ValidateParameters(inputs: Seq[Val[_]], implicit val newFile: TmpDirectory, implicit val fileService: FileService)
......@@ -62,14 +66,7 @@ object FromContextHook {
val fcp = FromContextHook.Parameters(
context,
cache = executionContext.cache,
preference = executionContext.preference,
threadProvider = executionContext.threadProvider,
fileService = executionContext.fileService,
workspace = executionContext.workspace,
outputRedirection = executionContext.outputRedirection,
loggerService = executionContext.loggerService,
random = executionContext.random,
newFile = executionContext.newFile)
executionContext = executionContext)
f(fcp)
}
......
......@@ -21,6 +21,7 @@ import org.openmole.core.context.Context
import org.openmole.core.expansion.FromContext
import org.openmole.core.fileservice.FileService
import org.openmole.core.preference.Preference
import org.openmole.core.serializer.SerializerService
import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.workflow.builder.{ InfoConfig, InputOutputConfig }
import org.openmole.core.workflow.tools._
......@@ -39,7 +40,8 @@ case class HookExecutionContext(
implicit val outputRedirection: OutputRedirection,
implicit val loggerService: LoggerService,
implicit val random: RandomProvider,
implicit val newFile: TmpDirectory)
implicit val newFile: TmpDirectory,
implicit val serializerService: SerializerService)
trait Hook <: Name {
......
......@@ -207,7 +207,8 @@ object MoleExecution extends JavaLogger {
loggerService = loggerService,
cache = subMoleExecutionState.moleExecution.keyValueCache,
lockRepository = subMoleExecutionState.moleExecution.lockRepository,
moleExecution = Some(subMoleExecutionState.moleExecution)
moleExecution = Some(subMoleExecutionState.moleExecution),
serializerService = serializerService
)
val result = moleJob.perform(taskContext)
......@@ -265,7 +266,8 @@ object MoleExecution extends JavaLogger {
outputRedirection = services.outputRedirection,
loggerService = services.loggerService,
random = services.newRandom,
newFile = services.tmpDirectory)
newFile = services.tmpDirectory,
serializerService = services.serializerService)
}
h.perform(ctxForHooks, toHookExecutionContext(subMoleExecutionState.moleExecution.keyValueCache, subMoleExecutionState.moleExecution.executionContext))
......@@ -452,7 +454,8 @@ object MoleExecution extends JavaLogger {
loggerService = loggerService,
cache = moleExecution.keyValueCache,
lockRepository = moleExecution.lockRepository,
moleExecution = Some(moleExecution)
moleExecution = Some(moleExecution),
serializerService = serializerService
)
)
}
......
......@@ -20,6 +20,7 @@ package org.openmole.core.workflow.mole
import org.openmole.core.event.EventDispatcher
import org.openmole.core.fileservice.{ FileService, FileServiceCache }
import org.openmole.core.preference.Preference
import org.openmole.core.serializer.SerializerService
import org.openmole.core.threadprovider._
import org.openmole.core.workspace._
import org.openmole.tool.cache._
......@@ -56,7 +57,7 @@ object MoleServices {
applicationExecutionDirectory: File,
moleExecutionDirectory: Option[File] = None,
outputRedirection: Option[OutputRedirection] = None,
seed: Option[Long] = None)(implicit preference: Preference, seeder: Seeder, threadProvider: ThreadProvider, eventDispatcher: EventDispatcher, _newFile: TmpDirectory, fileService: FileService, workspace: Workspace, _outputRedirection: OutputRedirection, loggerService: LoggerService) = {
seed: Option[Long] = None)(implicit preference: Preference, seeder: Seeder, threadProvider: ThreadProvider, eventDispatcher: EventDispatcher, _newFile: TmpDirectory, fileService: FileService, workspace: Workspace, _outputRedirection: OutputRedirection, loggerService: LoggerService, serializerService: SerializerService) = {
val executionDirectory = moleExecutionDirectory.getOrElse(applicationExecutionDirectory.newDir("execution"))
new MoleServices(applicationExecutionDirectory, executionDirectory)(
preference = preference,
......@@ -68,7 +69,8 @@ object MoleServices {
fileService = fileService,
fileServiceCache = FileServiceCache(),
outputRedirection = outputRedirection.getOrElse(_outputRedirection),
loggerService = loggerService
loggerService = loggerService,
serializerService = serializerService
)
}
......@@ -86,7 +88,8 @@ object MoleServices {
fileServiceCache: FileServiceCache = moleServices.fileServiceCache,
workspace: Workspace = moleServices.workspace,
outputRedirection: OutputRedirection = moleServices.outputRedirection,
loggerService: LoggerService = moleServices.loggerService) =
loggerService: LoggerService = moleServices.loggerService,
serializerService: SerializerService = moleServices.serializerService) =
new MoleServices(moleServices.applicationExecutionDirectory, moleServices.moleExecutionDirectory)(
preference = preference,
seeder = seeder,
......@@ -97,7 +100,8 @@ object MoleServices {
fileService = fileService,
fileServiceCache = fileServiceCache,
outputRedirection = outputRedirection,
loggerService = loggerService
loggerService = loggerService,
serializerService = serializerService
)
}
......@@ -125,7 +129,8 @@ class MoleServices(val applicationExecutionDirectory: File, val moleExecutionDir
val fileService: FileService,
val fileServiceCache: FileServiceCache,
val outputRedirection: OutputRedirection,
val loggerService: LoggerService
val loggerService: LoggerService,
val serializerService: SerializerService
) {
def newRandom = Lazy(seeder.newRNG)
implicit lazy val defaultRandom = newRandom
......
......@@ -117,6 +117,7 @@ object MoleTask {
import executionContext.workspace
import executionContext.outputRedirection
import executionContext.loggerService
import executionContext.serializerService
val localEnvironment =
LocalEnvironment(1, executionContext.localEnvironment.deinterleave)
......
......@@ -23,6 +23,7 @@ import org.openmole.core.context._
import org.openmole.core.expansion.FromContext
import org.openmole.core.fileservice.FileService
import org.openmole.core.preference.Preference
import org.openmole.core.serializer.SerializerService
import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.tools.obj.Id
import org.openmole.core.workflow.builder.{ DefinitionScope, InfoConfig, InputOutputConfig }
......@@ -63,6 +64,7 @@ case class TaskExecutionContext(
implicit val workspace: Workspace,
implicit val outputRedirection: OutputRedirection,
implicit val loggerService: LoggerService,
implicit val serializerService: SerializerService,
cache: KeyValueCache,
lockRepository: LockRepository[LockKey],
moleExecution: Option[MoleExecution] = None)
......
......@@ -7,7 +7,7 @@ import org.openmole.plugin.tool.json._
object JSONOutputFormat {
implicit def outputFormat = new OutputFormat[JSONOutputFormat, Any] {
override def write(format: JSONOutputFormat, output: WritableOutput, content: OutputContent, method: Any): FromContext[Unit] = FromContext { p
override def write(executionContext: HookExecutionContext)(format: JSONOutputFormat, output: WritableOutput, content: OutputContent, method: Any): FromContext[Unit] = FromContext { p
import p._
import org.json4s._
import org.json4s.jackson.JsonMethods._
......
......@@ -2,28 +2,55 @@ package org.openmole.plugin.hook.omr
import org.openmole.core.dsl._
import org.openmole.core.dsl.extension._
import org.openmole.core.serializer.SerializerService
import org.openmole.core.workflow.format.CSVOutputFormat
import org.openmole.core.workflow.hook.FromContextHook
import org.openmole.plugin.tool.json._
import io.circe._
import org.openmole.core.workflow.format.OutputFormat.{ PlainContent, SectionContent }
import org.openmole.core.workflow.format.WritableOutput.Store
import org.openmole.plugin.hook.json.JSONOutputFormat
object OMROutputFormat {
implicit def outputFormat: OutputFormat[OMROutputFormat, Any] = new OutputFormat[OMROutputFormat, Any] {
override def write(format: OMROutputFormat, output: WritableOutput, content: OutputContent, method: Any): FromContext[Unit] = FromContext { p
implicit def outputFormat[MD](implicit encoder: Encoder[MD]): OutputFormat[OMROutputFormat, MD] = new OutputFormat[OMROutputFormat, MD] {
override def write(executionContext: HookExecutionContext)(format: OMROutputFormat, output: WritableOutput, content: OutputContent, method: MD): FromContext[Unit] = FromContext { p
import p._
import org.json4s._
import org.json4s.jackson.JsonMethods._
implicit val formats = DefaultFormats
output match {
case WritableOutput.Display(stream) => implicitly[OutputFormat[CSVOutputFormat, Any]].write(CSVOutputFormat(), output, content, method)
case WritableOutput.Store(file) =>
import org.openmole.tool.tar._
// file.from(context).withTarGZOutputStream {
//
// }
case WritableOutput.Display(stream) implicitly[OutputFormat[CSVOutputFormat, Any]].write(executionContext)(CSVOutputFormat(), output, content, method).from(context)
case WritableOutput.Store(file)
import io.circe.generic.auto._, io.circe.parser._, io.circe.syntax._
content match {
case PlainContent(variables, name)
val (f, m) =
name match {
case Some(n) (file / "data" / s"${n.from(context)}.json.gz", file / "method.omr")
case None (file / "data.json.gz", file / "method.omr")
}
f.from(context).withPrintStream(append = false, create = true, gz = true) { ps
ps.print(compact(render(variablesToJValue(variables))))
}
m.from(context).withPrintStream(create = true, gz = true)(_.print(method.asJson.noSpaces))
case sections: SectionContent
def sectionContent(sections: SectionContent) =
JObject(
sections.sections.map { section section.name.from(context) -> variablesToJValue(section.variables) }.toList
)
val (f, m) = (file / "data.json.gz", file / "method.omr")
f.from(context).withPrintStream(append = false, create = true, gz = true) { ps
ps.print(compact(render(sectionContent(sections))))
}
m.from(context).withPrintStream(create = true, gz = true)(_.print(method.asJson.noSpaces))
}
}
}
......
package org.openmole.plugin.method.evolution
import org.openmole.core.context.Val
import org.openmole.core.expansion.FromContext
import io.circe._
import io.circe.generic.extras.auto._
import io.circe.generic.extras.semiauto._
import io.circe.generic.extras.Configuration
import org.openmole.core.tools.io.Prettifier._
object Metadata {
def method = "evolution"
implicit val genDevConfig: Configuration = Configuration.default.withDiscriminator("implementation")
implicit val metadataEncoder: Encoder[Metadata] = deriveConfiguredEncoder[Metadata]
case class nsga2(genome: Seq[GenomeBoundData], generation: Long, frequency: Option[Long], method: String = method) extends Metadata
case object none extends Metadata
sealed trait GenomeBoundData
object GenomeBoundData {
import Genome._
def apply(b: GenomeBound) = FromContext { p
import p._
b match {
case b: GenomeBound.ScalarDouble bound(b.v.name, b.low.from(context).prettify(), b.high.from(context).prettify())
case b: GenomeBound.ScalarInt bound(b.v.name, b.low.from(context).prettify(), b.high.from(context).prettify())
case b: GenomeBound.SequenceOfDouble bound(b.v.name, b.low.from(context).prettify(), b.high.from(context).prettify())
case b: GenomeBound.SequenceOfInt bound(b.v.name, b.low.from(context).prettify(), b.high.from(context).prettify())
case b: GenomeBound.Enumeration[_] enumeration(b.v.name, b.values.prettify())
case b: GenomeBound.SequenceOfEnumeration[_] enumeration(b.v.name, b.values.prettify())
}
}
case class bound(value: String, low: String, high: String) extends GenomeBoundData
case class enumeration(value: String, values: String) extends GenomeBoundData
}
}
sealed trait Metadata
\ No newline at end of file
......@@ -21,7 +21,7 @@ import cats.implicits._
import monocle.macros.GenLens
import org.openmole.core.dsl._
import org.openmole.core.dsl.extension._
import org.openmole.plugin.method.evolution.Genome.Suggestion
import org.openmole.plugin.method.evolution.Genome.{ Suggestion }
import squants.time.Time
import scala.language.higherKinds
......@@ -29,6 +29,7 @@ import scala.language.higherKinds
object NSGA2 {
object stochastic {
case class SavedGeneration(generation: Long, objectives: Vector[SavedObjective])
case class SavedObjective(objectives: Vector[Double], samples: Int)
......@@ -74,7 +75,6 @@ object NSGA2 {
def sManifest = implicitly
def operations(om: DeterministicParams) = new Ops {
def startTimeLens = GenLens[EvolutionState[Unit]](_.startTime)
def generationLens = GenLens[EvolutionState[Unit]](_.generation)
......@@ -153,6 +153,16 @@ object NSGA2 {
def operations(om: StochasticParams) = new Ops {
override def metadata(generation: Long, frequency: Option[Long]) = FromContext { p
import p._
import Metadata._
nsga2(
om.genome.map(GenomeBoundData(_).from(context)),
generation = generation,
frequency = frequency
)
}
def startTimeLens = GenLens[S](_.startTime)
def generationLens = GenLens[S](_.generation)
......
......@@ -234,7 +234,6 @@ object NichedNSGA2 {
def sManifest = implicitly
def operations(om: DeterministicParams) = new Ops {
def startTimeLens = GenLens[S](_.startTime)
def generationLens = GenLens[S](_.generation)
......@@ -339,7 +338,6 @@ object NichedNSGA2 {
def sManifest = implicitly
def operations(om: StochasticParams) = new Ops {
def startTimeLens = GenLens[S](_.startTime)
def generationLens = GenLens[S](_.generation)
......
......@@ -238,7 +238,6 @@ object PSE {
def sManifest = implicitly
def operations(om: DeterministicParams) = new Ops {
def startTimeLens = GenLens[S](_.startTime)
def generationLens = GenLens[S](_.generation)
......@@ -328,7 +327,6 @@ object PSE {
def sManifest = implicitly
def operations(om: StochasticParams) = new Ops {
def startTimeLens = GenLens[S](_.startTime)
def generationLens = GenLens[S](_.generation)
......