[Plugin] enh: first version of GAMATask based on container

libraries/build.sbt

@@ -235,7 +235,7 @@ lazy val mgo = OsgiProject(dir, "mgo", exports = Seq("mgo.*", "freestyle.*"), im
   version := mgoVersion) dependsOn(monocle, math, cats, squants) settings(settings: _*)
 
 
-lazy val containerVersion = "1.9"
+lazy val containerVersion = "1.10"
 
 lazy val container = OsgiProject(dir, "container", exports = Seq("container.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "!com.github.luben.*", "!org.apache.avalon.*", "!org.apache.log.*", "!org.brotli.dec.*", "!javax.*","*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "!io.circe.*", "!shapeless.*", "*")) settings(
   libraryDependencies += "org.openmole" %% "container" % containerVersion,

openmole/build.sbt

@@ -508,7 +508,7 @@ lazy val fileSource = OsgiProject(pluginDir, "org.openmole.plugin.source.file",
 
 /* Task */
 
-def allTask = Seq(toolsTask, external, netLogo, netLogo5, netLogo6, jvm, scala, template, systemexec, container, r, scilab, python, timing)
+def allTask = Seq(toolsTask, external, netLogo, netLogo5, netLogo6, jvm, scala, template, systemexec, container, r, scilab, python, gama, timing)
 
 lazy val toolsTask = OsgiProject(pluginDir, "org.openmole.plugin.task.tools", imports = Seq("*")) dependsOn (openmoleDSL) settings (pluginSettings: _*)
 
@@ -540,6 +540,8 @@ lazy val scilab = OsgiProject(pluginDir, "org.openmole.plugin.task.scilab", impo
 
 lazy val python = OsgiProject(pluginDir, "org.openmole.plugin.task.python", imports = Seq("*")) dependsOn (container, json) settings (pluginSettings: _*)
 
+lazy val gama = OsgiProject(pluginDir, "org.openmole.plugin.task.gama", imports = Seq("*")) dependsOn (container) settings (pluginSettings: _*)
+
 lazy val timing = OsgiProject(pluginDir, "org.openmole.plugin.task.timing", imports = Seq("*")) dependsOn (openmoleDSL) settings (pluginSettings: _*)
 
 

openmole/core/org.openmole.core.workflow/src/main/scala/org/openmole/core/workflow/validation/package.scala

@@ -7,6 +7,7 @@ import org.openmole.core.workspace.TmpDirectory
 package object validation {
   trait Validate {
     def apply(implicit newFile: TmpDirectory, fileService: FileService): Seq[Throwable]
+    def ++(v: Validate) = Validate(this, v)
   }
 
   object Validate {

openmole/core/org.openmole.core.workspace/src/main/scala/org/openmole/core/workspace/TmpDirectory.scala

@@ -34,7 +34,7 @@ case class TmpDirectory(directory: File) {
   def withTmpDir[T](f: File ⇒ T): T = {
     val file = newFile()
     try {
-      file.mkdir()
+      file.mkdirs()
       f(file)
     }
     finally file.recursiveDelete

openmole/plugins/org.openmole.plugin.task.container/src/main/scala/org/openmole/plugin/task/container/ContainerTask.scala

@@ -180,7 +180,7 @@ object ContainerTask {
       containerPoolKey = containerPoolKey)
   }
 
-  def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String])(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = {
+  def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String], volumes: Seq[(String, String)] = Seq.empty)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = {
     def cacheId(image: ContainerImage): Seq[String] =
       image match {
         case image: DockerImage ⇒ Seq(image.image, image.tag, image.registry)
@@ -200,7 +200,7 @@ object ContainerTask {
       else {
         val containerDirectory = cacheDirectory / "fs"
         val img = localImage(image, containerDirectory)
-        val installedImage = executeInstall(containerSystem, img, install)
+        val installedImage = executeInstall(containerSystem, img, install, volumes = volumes)
         serializerService.serialize(installedImage, serializedFlatImage)
         //OutputManager.systemOutput.println("created " + serializedFlatImage + " " + serializedFlatImage.exists)
         installedImage
@@ -208,10 +208,10 @@ object ContainerTask {
     }
   }
 
-  def executeInstall(containerSystem: ContainerSystem, image: _root_.container.FlatImage, install: Seq[String])(implicit tmpDirectory: TmpDirectory, outputRedirection: OutputRedirection) =
+  def executeInstall(containerSystem: ContainerSystem, image: _root_.container.FlatImage, install: Seq[String], volumes: Seq[(String, String)])(implicit tmpDirectory: TmpDirectory, outputRedirection: OutputRedirection) =
     if (install.isEmpty) image
     else {
-      val retCode = runCommandInContainer(containerSystem, image, install, output = outputRedirection.output, error = outputRedirection.error)
+      val retCode = runCommandInContainer(containerSystem, image, install, output = outputRedirection.output, error = outputRedirection.error, volumes = volumes)
       if (retCode != 0) throw new UserBadDataError(s"Process exited a non 0 return code ($retCode)")
       image
     }

openmole/plugins/org.openmole.plugin.task.gama/src/main/scala/org/openmole/plugin/task/gama/Activator.scala

+/*
+ * Copyright (C) 2015 Romain Reuillon
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.openmole.plugin.task.gama
+
+import org.openmole.core.pluginmanager._
+import org.openmole.core.preference.ConfigurationInfo
+import org.osgi.framework.BundleContext
+
+class Activator extends PluginInfoActivator {
+
+  override def stop(context: BundleContext): Unit = {
+    PluginInfo.unregister(this)
+    ConfigurationInfo.unregister(this)
+  }
+
+  override def start(context: BundleContext): Unit = {
+    import org.openmole.core.pluginmanager.KeyWord._
+
+    val keyWords: Vector[KeyWord] =
+      Vector(
+        TaskKeyWord(objectName(GAMATask)))
+
+    PluginInfo.register(this, Vector(this.getClass.getPackage), keyWords = keyWords)
+    ConfigurationInfo.register(
+      this,
+      ConfigurationInfo.list()
+    )
+  }
+}
\ No newline at end of file

openmole/plugins/org.openmole.plugin.task.gama/src/main/scala/org/openmole/plugin/task/gama/GAMATask.scala

+package org.openmole.plugin.task.gama
+
+import monocle.macros._
+import org.openmole.core.dsl._
+import org.openmole.core.dsl.extension._
+import org.openmole.core.exception.{InternalProcessingError, UserBadDataError}
+import org.openmole.core.fileservice.FileService
+import org.openmole.core.networkservice.NetworkService
+import org.openmole.core.preference.Preference
+import org.openmole.core.serializer.SerializerService
+import org.openmole.core.threadprovider.ThreadProvider
+import org.openmole.core.workflow.builder._
+import org.openmole.core.workflow.task._
+import org.openmole.core.workflow.validation._
+import org.openmole.core.workspace.{TmpDirectory, Workspace}
+import org.openmole.plugin.task.container
+import org.openmole.plugin.task.container._
+import org.openmole.plugin.task.external._
+import org.openmole.tool.outputredirection.OutputRedirection
+
+import scala.xml.XML
+
+object GAMATask {
+
+  implicit def isTask: InputOutputBuilder[GAMATask] = InputOutputBuilder(GAMATask.config)
+  implicit def isExternal: ExternalBuilder[GAMATask] = ExternalBuilder(GAMATask.external)
+  implicit def isInfo = InfoBuilder(info)
+  implicit def isMapped = MappedInputOutputBuilder(GAMATask.mapped)
+
+  def gamaImage(version: String) = DockerImage("gamaplatform/gama", version)
+
+  def inputXML = "/_model_input_.xml"
+  def workspaceDirectory = "/_workspace_"
+
+  def volumes(
+    workspace: File,
+    model:     OptionalArgument[String]) =
+    model.option match {
+      case Some(model) ⇒ (model, Seq(workspace -> workspaceDirectory))
+      case None        ⇒ (workspace.getName, Seq(workspace -> s"$workspaceDirectory/${workspace.getName}"))
+    }
+
+  def prepare(
+    workspace:              File,
+    model:                  OptionalArgument[String],
+    experiment:             String,
+    install:                Seq[String],
+    installContainerSystem: ContainerSystem,
+    version:                String)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, _workspace: Workspace) = {
+
+    val (modelName, volumesValue) = volumes(workspace, model)
+
+    def installCommands =
+      install ++ Seq(s"gama-headless -xml $experiment $workspaceDirectory/$modelName $inputXML", s"ls ${inputXML}")
+
+    ContainerTask.prepare(installContainerSystem, gamaImage(version), installCommands, volumesValue.map { case (lv, cv) ⇒ lv.getAbsolutePath -> cv })
+  }
+
+  def apply(
+    workspace:              File,
+    experiment:             String,
+    finalStep:              FromContext[Int],
+    model:                  OptionalArgument[String]      = None,
+    seed:                   OptionalArgument[Val[Long]]   = None,
+    frameRate:              OptionalArgument[Int]         = None,
+    install:                Seq[String]                   = Seq.empty,
+    version:                String                        = "1.8.0",
+    errorOnReturnValue:     Boolean                       = true,
+    returnValue:            OptionalArgument[Val[Int]]    = None,
+    stdOut:                 OptionalArgument[Val[String]] = None,
+    stdErr:                 OptionalArgument[Val[String]] = None,
+    environmentVariables:   Seq[EnvironmentVariable]      = Vector.empty,
+    hostFiles:              Seq[HostFile]                 = Vector.empty,
+    workDirectory:          OptionalArgument[String]      = None,
+    containerSystem:        ContainerSystem               = ContainerSystem.default,
+    installContainerSystem: ContainerSystem               = ContainerSystem.default)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: TmpDirectory, _workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService, serializerService: SerializerService): GAMATask = {
+
+    (model.option.isDefined, workspace.isDirectory) match {
+      case (false, true) ⇒ throw new UserBadDataError(s"""$workspace is a directory, in this case you must specify you model path, model = "model.gaml"""")
+      case (true, false) ⇒ throw new UserBadDataError(s"""$workspace is a file in this case you cannot provide a model path (model = "$model")""")
+      case (true, true) if !(workspace / model.get).exists() ⇒ throw new UserBadDataError(s"The model file you specify does not exist: ${workspace / model.get}")
+      case _ ⇒
+    }
+
+    val preparedImage = prepare(workspace, model, experiment, install, installContainerSystem, version)
+
+    GAMATask(
+      workspace = workspace,
+      model = model,
+      experiment = experiment,
+      finalStep = finalStep,
+      seed = seed,
+      frameRate = frameRate,
+      image = preparedImage,
+      errorOnReturnValue = errorOnReturnValue,
+      returnValue = returnValue,
+      stdOut = stdOut,
+      stdErr = stdErr,
+      hostFiles = hostFiles,
+      environmentVariables = environmentVariables,
+      containerSystem = containerSystem,
+      config = InputOutputConfig(),
+      external = External(),
+      info = InfoConfig(),
+      mapped = MappedInputOutputConfig(),
+      version = version
+    ) set (
+        inputs += (seed.option.toSeq: _*),
+        outputs += (Seq(returnValue.option, stdOut.option, stdErr.option).flatten: _*)
+      )
+  }
+
+
+  def modifyInputXML(values: Map[String, String], finalStep: Int, seed: Long, frameRate: Option[Int]) = {
+    import xml._
+    import xml.transform._
+
+    def value(n: Node): Option[String] =
+      if (n.label != "Parameter") None
+      else n.attribute("var").flatMap(_.headOption.map(_.text)).flatMap(values.get) orElse n.attribute("name").flatMap(_.headOption.map(_.text)).flatMap(values.get)
+
+    val rewrite =
+      new RewriteRule {
+        override def transform(n: Node): Seq[Node] =
+          (n, value(n), frameRate) match {
+            case (n: Elem, Some(v), _) ⇒ n.copy(attributes = n.attributes.remove("value").append(Attribute(null, "value", v, Null)))
+            case (n: Elem, _, _) if n.label == "Simulation" =>
+              n.copy(attributes =
+                n.attributes
+                  .remove("finalStep").append(Attribute(null, "finalStep", finalStep.toString, Null))
+                  .remove("seed").append(Attribute(null, "seed", seed.toDouble.toString, Null))
+              )
+            case (n: Elem, _, Some(f)) if n.label == "Output" =>
+              n.copy(attributes = n.attributes.remove("framerate").append(Attribute(null, "framerate", f.toString, Null)))
+            case _                  ⇒ n
+          }
+      }
+
+    new RuleTransformer(rewrite)
+  }
+
+}
+
+@Lenses case class GAMATask(
+  workspace:            File,
+  model:                OptionalArgument[String],
+  experiment:           String,
+  finalStep:            FromContext[Int],
+  seed:                 OptionalArgument[Val[Long]],
+  frameRate:            OptionalArgument[Int],
+  image:                PreparedImage,
+  errorOnReturnValue:   Boolean,
+  returnValue:          Option[Val[Int]],
+  stdOut:               Option[Val[String]],
+  stdErr:               Option[Val[String]],
+  hostFiles:            Seq[HostFile],
+  environmentVariables: Seq[EnvironmentVariable],
+  containerSystem:      ContainerSystem,
+  config:               InputOutputConfig,
+  external:             External,
+  info:                 InfoConfig,
+  mapped:               MappedInputOutputConfig,
+  version:              String) extends Task with ValidateTask {
+
+  lazy val containerPoolKey = ContainerTask.newCacheKey
+
+  override def validate =
+    container.validateContainer(Vector(), environmentVariables, external, inputs) ++ {
+      if ((XML.loadFile(image.file / _root_.container.FlatImage.rootfsName / GAMATask.inputXML) \ "Simulation").isEmpty) Seq(new UserBadDataError(s"Experiment ${experiment} has not been found, make sure it is defined in your gaml file"))
+      else Seq()
+    }
+
+  override def process(executionContext: TaskExecutionContext) = FromContext { p ⇒
+    import p._
+
+    newFile.withTmpFile("inputs", ".xml") { inputFile ⇒
+      val seedValue = seed.map(_.from(context)).getOrElse(random().nextLong)
+
+      def inputMap = mapped.inputs.map { m ⇒ m.name -> context(m.v).toString }.toMap
+      val inputXML = GAMATask.modifyInputXML(inputMap, finalStep.from(context), seedValue, frameRate.option).transform(XML.loadFile(image.file / _root_.container.FlatImage.rootfsName / GAMATask.inputXML))
+      def inputFileName = "/_inputs_openmole_.xml"
+      def outputDirectory = "/_output_"
+
+      inputFile.content =
+        s"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>${inputXML.mkString("")}"""
+
+
+      val (_, volumes) = GAMATask.volumes(workspace, model)
+
+      def launchCommand = s"gama-headless -hpc 1 $inputFileName $outputDirectory"
+
+        newFile.withTmpDir { tmpOutputDirectory =>
+          def containerTask =
+            ContainerTask(
+              containerSystem = containerSystem,
+              image = image,
+              command = launchCommand,
+              workDirectory = None,
+              errorOnReturnValue = errorOnReturnValue,
+              returnValue = returnValue,
+              hostFiles = hostFiles,
+              environmentVariables = environmentVariables,
+              reuseContainer = true,
+              stdOut = stdOut,
+              stdErr = stdErr,
+              config = config,
+              external = external,
+              info = info,
+              containerPoolKey = containerPoolKey) set(
+              resources += (inputFile, inputFileName, true),
+              volumes.map { case (lv, cv) ⇒ resources +=[ContainerTask](lv, cv, true) },
+              resources += (tmpOutputDirectory, outputDirectory, true)
+            )
+
+          val resultContext = containerTask.process(executionContext).from(context)
+
+          frameRate.option match {
+            case None =>
+              import xml._
+
+              def toVariable(v: Val[_], value: String) =
+                v match {
+                  case Val.caseInt(v) => Variable(v, value.toInt)
+                  case Val.caseDouble(v) => Variable(v, value.toDouble)
+                  case Val.caseString(v) => Variable(v, value)
+                  case Val.caseBoolean(v) => Variable(v, value.toBoolean)
+                  case _ => throw new UserBadDataError(s"Unsupported type of output variable $v")
+                }
+
+              val outputs = Map[String, Val[_]]() ++ mapped.outputs.map { m => (m.name, m.v) }
+              def outputValue(e: Elem) =
+                for {
+                  a <- e.attribute("name").flatMap(_.headOption)
+                  value <- outputs.get(a.text)
+                } yield toVariable(value, e.child.text)
+
+              def extractOutputs(n: Node) =
+                (n \ "Variable").flatMap {
+                  case e: Elem => outputValue(e)
+                  case _ => None
+                }
+
+              val simulationOutput = XML.loadFile(tmpOutputDirectory / "simulation-outputs0.xml") \ "Step"
+              resultContext ++ extractOutputs(simulationOutput.last)
+            case Some(f) =>
+              import xml._
+
+              def toVariable(v: Val[_], value: Array[String]) =
+                v match {
+                  case Val.caseArrayInt(v) => Variable(v, value.map(_.toInt))
+                  case Val.caseArrayDouble(v) => Variable(v, value.map(_.toDouble))
+                  case Val.caseArrayString(v) => Variable(v, value)
+                  case Val.caseArrayBoolean(v) => Variable(v, value.map(_.toBoolean))
+                  case _ => throw new UserBadDataError(s"Unsupported type of output variable $v")
+                }
+
+              def outputValue(e: Elem, name: String) =
+                for {
+                  a <- e.attribute("name").flatMap(_.headOption)
+                  if a.text == name
+                } yield e.child.text
+
+              val simulationOutput = XML.loadFile(tmpOutputDirectory / "simulation-outputs0.xml") \ "Step"
+
+              resultContext ++ mapped.outputs.map { m =>
+                val values =
+                  for {
+                    o <- simulationOutput
+                    v <- o \ "Variable"
+                  } yield
+                    v match {
+                      case o: Elem => outputValue(o, m.name)
+                      case _ => None
+                    }
+
+                toVariable(m.v, values.flatten.toArray)
+              }
+          }
+        }
+    }
+
+  }
+}

openmole/plugins/org.openmole.plugin.task.python/src/main/scala/org/openmole/plugin/task/python/PythonTask.scala

@@ -33,7 +33,7 @@ object PythonTask {
 
     def installCommands(install: Seq[String], libraries: Seq[String], major: Int): Vector[String] = {
       // need to install pip2 in case of python 2
-      val effintsall = install++(if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty)
+      val effintsall = install ++ (if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty)
       (effintsall ++ libraries.map { l ⇒ "pip"+major+" install " + l }).toVector
     }
 
@@ -43,7 +43,6 @@ object PythonTask {
       major:                Int = 3,
       libraries:            Seq[String]                        = Seq.empty,
       install:              Seq[String]                        = Seq.empty,
-      forceUpdate:          Boolean                            = false,
       workDirectory:        OptionalArgument[String]           = None,
       hostFiles:            Seq[HostFile]                      = Vector.empty,
       environmentVariables: Seq[EnvironmentVariable] = Vector.empty,

openmole/plugins/org.openmole.plugin.task.r/src/main/scala/org/openmole/plugin/task/r/RTask.scala

@@ -55,7 +55,6 @@ object RTask {
     script:               RunnableScript,
     install:              Seq[String]                        = Seq.empty,
     libraries:            Seq[InstallCommand]                = Seq.empty,
-    forceUpdate:          Boolean                            = false,
     version:              String                             = "3.6.2",
     errorOnReturnValue:   Boolean                            = true,
     returnValue:          OptionalArgument[Val[Int]]         = None,

openmole/plugins/org.openmole.plugin.task.scilab/src/main/scala/org/openmole/plugin/task/scilab/ScilabTask.scala

@@ -33,7 +33,6 @@ object ScilabTask {
   def apply(
     script:                 RunnableScript,
     install:                Seq[String]                   = Seq.empty,
-    forceUpdate:            Boolean                       = false,
     version:                String                        = "6.1.0",
     errorOnReturnValue:     Boolean                       = true,
     returnValue:            OptionalArgument[Val[Int]]    = None,

openmole/project/Libraries.scala

@@ -8,7 +8,7 @@ object Libraries {
 
   lazy val gridscaleVersion = "2.28"
   lazy val sshjVersion = "0.27.0"
-  lazy val containerVersion = "1.9"
+  lazy val containerVersion = "1.10"
   lazy val mgoVersion = "3.43"
   lazy val bouncyCastleVersion = "1.64"
   lazy val d3Version = "3.5.12"