Commit c638826d authored by Romain Reuillon's avatar Romain Reuillon
Browse files

[Plugin] enh: first version of GAMATask based on container

parent ee98155b
......@@ -235,7 +235,7 @@ lazy val mgo = OsgiProject(dir, "mgo", exports = Seq("mgo.*", "freestyle.*"), im
version := mgoVersion) dependsOn(monocle, math, cats, squants) settings(settings: _*)
lazy val containerVersion = "1.9"
lazy val containerVersion = "1.10"
lazy val container = OsgiProject(dir, "container", exports = Seq("container.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "!com.github.luben.*", "!org.apache.avalon.*", "!org.apache.log.*", "!org.brotli.dec.*", "!javax.*","*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "!io.circe.*", "!shapeless.*", "*")) settings(
libraryDependencies += "org.openmole" %% "container" % containerVersion,
......
......@@ -508,7 +508,7 @@ lazy val fileSource = OsgiProject(pluginDir, "org.openmole.plugin.source.file",
/* Task */
def allTask = Seq(toolsTask, external, netLogo, netLogo5, netLogo6, jvm, scala, template, systemexec, container, r, scilab, python, timing)
def allTask = Seq(toolsTask, external, netLogo, netLogo5, netLogo6, jvm, scala, template, systemexec, container, r, scilab, python, gama, timing)
lazy val toolsTask = OsgiProject(pluginDir, "org.openmole.plugin.task.tools", imports = Seq("*")) dependsOn (openmoleDSL) settings (pluginSettings: _*)
......@@ -540,6 +540,8 @@ lazy val scilab = OsgiProject(pluginDir, "org.openmole.plugin.task.scilab", impo
lazy val python = OsgiProject(pluginDir, "org.openmole.plugin.task.python", imports = Seq("*")) dependsOn (container, json) settings (pluginSettings: _*)
lazy val gama = OsgiProject(pluginDir, "org.openmole.plugin.task.gama", imports = Seq("*")) dependsOn (container) settings (pluginSettings: _*)
lazy val timing = OsgiProject(pluginDir, "org.openmole.plugin.task.timing", imports = Seq("*")) dependsOn (openmoleDSL) settings (pluginSettings: _*)
......
......@@ -7,6 +7,7 @@ import org.openmole.core.workspace.TmpDirectory
package object validation {
trait Validate {
def apply(implicit newFile: TmpDirectory, fileService: FileService): Seq[Throwable]
def ++(v: Validate) = Validate(this, v)
}
object Validate {
......
......@@ -34,7 +34,7 @@ case class TmpDirectory(directory: File) {
def withTmpDir[T](f: File T): T = {
val file = newFile()
try {
file.mkdir()
file.mkdirs()
f(file)
}
finally file.recursiveDelete
......
......@@ -180,7 +180,7 @@ object ContainerTask {
containerPoolKey = containerPoolKey)
}
def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String])(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = {
def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String], volumes: Seq[(String, String)] = Seq.empty)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = {
def cacheId(image: ContainerImage): Seq[String] =
image match {
case image: DockerImage Seq(image.image, image.tag, image.registry)
......@@ -200,7 +200,7 @@ object ContainerTask {
else {
val containerDirectory = cacheDirectory / "fs"
val img = localImage(image, containerDirectory)
val installedImage = executeInstall(containerSystem, img, install)
val installedImage = executeInstall(containerSystem, img, install, volumes = volumes)
serializerService.serialize(installedImage, serializedFlatImage)
//OutputManager.systemOutput.println("created " + serializedFlatImage + " " + serializedFlatImage.exists)
installedImage
......@@ -208,10 +208,10 @@ object ContainerTask {
}
}
def executeInstall(containerSystem: ContainerSystem, image: _root_.container.FlatImage, install: Seq[String])(implicit tmpDirectory: TmpDirectory, outputRedirection: OutputRedirection) =
def executeInstall(containerSystem: ContainerSystem, image: _root_.container.FlatImage, install: Seq[String], volumes: Seq[(String, String)])(implicit tmpDirectory: TmpDirectory, outputRedirection: OutputRedirection) =
if (install.isEmpty) image
else {
val retCode = runCommandInContainer(containerSystem, image, install, output = outputRedirection.output, error = outputRedirection.error)
val retCode = runCommandInContainer(containerSystem, image, install, output = outputRedirection.output, error = outputRedirection.error, volumes = volumes)
if (retCode != 0) throw new UserBadDataError(s"Process exited a non 0 return code ($retCode)")
image
}
......
/*
* Copyright (C) 2015 Romain Reuillon
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.openmole.plugin.task.gama
import org.openmole.core.pluginmanager._
import org.openmole.core.preference.ConfigurationInfo
import org.osgi.framework.BundleContext
class Activator extends PluginInfoActivator {
override def stop(context: BundleContext): Unit = {
PluginInfo.unregister(this)
ConfigurationInfo.unregister(this)
}
override def start(context: BundleContext): Unit = {
import org.openmole.core.pluginmanager.KeyWord._
val keyWords: Vector[KeyWord] =
Vector(
TaskKeyWord(objectName(GAMATask)))
PluginInfo.register(this, Vector(this.getClass.getPackage), keyWords = keyWords)
ConfigurationInfo.register(
this,
ConfigurationInfo.list()
)
}
}
\ No newline at end of file
package org.openmole.plugin.task.gama
import monocle.macros._
import org.openmole.core.dsl._
import org.openmole.core.dsl.extension._
import org.openmole.core.exception.{InternalProcessingError, UserBadDataError}
import org.openmole.core.fileservice.FileService
import org.openmole.core.networkservice.NetworkService
import org.openmole.core.preference.Preference
import org.openmole.core.serializer.SerializerService
import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.task._
import org.openmole.core.workflow.validation._
import org.openmole.core.workspace.{TmpDirectory, Workspace}
import org.openmole.plugin.task.container
import org.openmole.plugin.task.container._
import org.openmole.plugin.task.external._
import org.openmole.tool.outputredirection.OutputRedirection
import scala.xml.XML
object GAMATask {
implicit def isTask: InputOutputBuilder[GAMATask] = InputOutputBuilder(GAMATask.config)
implicit def isExternal: ExternalBuilder[GAMATask] = ExternalBuilder(GAMATask.external)
implicit def isInfo = InfoBuilder(info)
implicit def isMapped = MappedInputOutputBuilder(GAMATask.mapped)
def gamaImage(version: String) = DockerImage("gamaplatform/gama", version)
def inputXML = "/_model_input_.xml"
def workspaceDirectory = "/_workspace_"
def volumes(
workspace: File,
model: OptionalArgument[String]) =
model.option match {
case Some(model) (model, Seq(workspace -> workspaceDirectory))
case None (workspace.getName, Seq(workspace -> s"$workspaceDirectory/${workspace.getName}"))
}
def prepare(
workspace: File,
model: OptionalArgument[String],
experiment: String,
install: Seq[String],
installContainerSystem: ContainerSystem,
version: String)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, _workspace: Workspace) = {
val (modelName, volumesValue) = volumes(workspace, model)
def installCommands =
install ++ Seq(s"gama-headless -xml $experiment $workspaceDirectory/$modelName $inputXML", s"ls ${inputXML}")
ContainerTask.prepare(installContainerSystem, gamaImage(version), installCommands, volumesValue.map { case (lv, cv) lv.getAbsolutePath -> cv })
}
def apply(
workspace: File,
experiment: String,
finalStep: FromContext[Int],
model: OptionalArgument[String] = None,
seed: OptionalArgument[Val[Long]] = None,
frameRate: OptionalArgument[Int] = None,
install: Seq[String] = Seq.empty,
version: String = "1.8.0",
errorOnReturnValue: Boolean = true,
returnValue: OptionalArgument[Val[Int]] = None,
stdOut: OptionalArgument[Val[String]] = None,
stdErr: OptionalArgument[Val[String]] = None,
environmentVariables: Seq[EnvironmentVariable] = Vector.empty,
hostFiles: Seq[HostFile] = Vector.empty,
workDirectory: OptionalArgument[String] = None,
containerSystem: ContainerSystem = ContainerSystem.default,
installContainerSystem: ContainerSystem = ContainerSystem.default)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: TmpDirectory, _workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService, serializerService: SerializerService): GAMATask = {
(model.option.isDefined, workspace.isDirectory) match {
case (false, true) throw new UserBadDataError(s"""$workspace is a directory, in this case you must specify you model path, model = "model.gaml"""")
case (true, false) throw new UserBadDataError(s"""$workspace is a file in this case you cannot provide a model path (model = "$model")""")
case (true, true) if !(workspace / model.get).exists() throw new UserBadDataError(s"The model file you specify does not exist: ${workspace / model.get}")
case _
}
val preparedImage = prepare(workspace, model, experiment, install, installContainerSystem, version)
GAMATask(
workspace = workspace,
model = model,
experiment = experiment,
finalStep = finalStep,
seed = seed,
frameRate = frameRate,
image = preparedImage,
errorOnReturnValue = errorOnReturnValue,
returnValue = returnValue,
stdOut = stdOut,
stdErr = stdErr,
hostFiles = hostFiles,
environmentVariables = environmentVariables,
containerSystem = containerSystem,
config = InputOutputConfig(),
external = External(),
info = InfoConfig(),
mapped = MappedInputOutputConfig(),
version = version
) set (
inputs += (seed.option.toSeq: _*),
outputs += (Seq(returnValue.option, stdOut.option, stdErr.option).flatten: _*)
)
}
def modifyInputXML(values: Map[String, String], finalStep: Int, seed: Long, frameRate: Option[Int]) = {
import xml._
import xml.transform._
def value(n: Node): Option[String] =
if (n.label != "Parameter") None
else n.attribute("var").flatMap(_.headOption.map(_.text)).flatMap(values.get) orElse n.attribute("name").flatMap(_.headOption.map(_.text)).flatMap(values.get)
val rewrite =
new RewriteRule {
override def transform(n: Node): Seq[Node] =
(n, value(n), frameRate) match {
case (n: Elem, Some(v), _) n.copy(attributes = n.attributes.remove("value").append(Attribute(null, "value", v, Null)))
case (n: Elem, _, _) if n.label == "Simulation" =>
n.copy(attributes =
n.attributes
.remove("finalStep").append(Attribute(null, "finalStep", finalStep.toString, Null))
.remove("seed").append(Attribute(null, "seed", seed.toDouble.toString, Null))
)
case (n: Elem, _, Some(f)) if n.label == "Output" =>
n.copy(attributes = n.attributes.remove("framerate").append(Attribute(null, "framerate", f.toString, Null)))
case _ n
}
}
new RuleTransformer(rewrite)
}
}
@Lenses case class GAMATask(
workspace: File,
model: OptionalArgument[String],
experiment: String,
finalStep: FromContext[Int],
seed: OptionalArgument[Val[Long]],
frameRate: OptionalArgument[Int],
image: PreparedImage,
errorOnReturnValue: Boolean,
returnValue: Option[Val[Int]],
stdOut: Option[Val[String]],
stdErr: Option[Val[String]],
hostFiles: Seq[HostFile],
environmentVariables: Seq[EnvironmentVariable],
containerSystem: ContainerSystem,
config: InputOutputConfig,
external: External,
info: InfoConfig,
mapped: MappedInputOutputConfig,
version: String) extends Task with ValidateTask {
lazy val containerPoolKey = ContainerTask.newCacheKey
override def validate =
container.validateContainer(Vector(), environmentVariables, external, inputs) ++ {
if ((XML.loadFile(image.file / _root_.container.FlatImage.rootfsName / GAMATask.inputXML) \ "Simulation").isEmpty) Seq(new UserBadDataError(s"Experiment ${experiment} has not been found, make sure it is defined in your gaml file"))
else Seq()
}
override def process(executionContext: TaskExecutionContext) = FromContext { p
import p._
newFile.withTmpFile("inputs", ".xml") { inputFile
val seedValue = seed.map(_.from(context)).getOrElse(random().nextLong)
def inputMap = mapped.inputs.map { m m.name -> context(m.v).toString }.toMap
val inputXML = GAMATask.modifyInputXML(inputMap, finalStep.from(context), seedValue, frameRate.option).transform(XML.loadFile(image.file / _root_.container.FlatImage.rootfsName / GAMATask.inputXML))
def inputFileName = "/_inputs_openmole_.xml"
def outputDirectory = "/_output_"
inputFile.content =
s"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>${inputXML.mkString("")}"""
val (_, volumes) = GAMATask.volumes(workspace, model)
def launchCommand = s"gama-headless -hpc 1 $inputFileName $outputDirectory"
newFile.withTmpDir { tmpOutputDirectory =>
def containerTask =
ContainerTask(
containerSystem = containerSystem,
image = image,
command = launchCommand,
workDirectory = None,
errorOnReturnValue = errorOnReturnValue,
returnValue = returnValue,
hostFiles = hostFiles,
environmentVariables = environmentVariables,
reuseContainer = true,
stdOut = stdOut,
stdErr = stdErr,
config = config,
external = external,
info = info,
containerPoolKey = containerPoolKey) set(
resources += (inputFile, inputFileName, true),
volumes.map { case (lv, cv) resources +=[ContainerTask](lv, cv, true) },
resources += (tmpOutputDirectory, outputDirectory, true)
)
val resultContext = containerTask.process(executionContext).from(context)
frameRate.option match {
case None =>
import xml._
def toVariable(v: Val[_], value: String) =
v match {
case Val.caseInt(v) => Variable(v, value.toInt)
case Val.caseDouble(v) => Variable(v, value.toDouble)
case Val.caseString(v) => Variable(v, value)
case Val.caseBoolean(v) => Variable(v, value.toBoolean)
case _ => throw new UserBadDataError(s"Unsupported type of output variable $v")
}
val outputs = Map[String, Val[_]]() ++ mapped.outputs.map { m => (m.name, m.v) }
def outputValue(e: Elem) =
for {
a <- e.attribute("name").flatMap(_.headOption)
value <- outputs.get(a.text)
} yield toVariable(value, e.child.text)
def extractOutputs(n: Node) =
(n \ "Variable").flatMap {
case e: Elem => outputValue(e)
case _ => None
}
val simulationOutput = XML.loadFile(tmpOutputDirectory / "simulation-outputs0.xml") \ "Step"
resultContext ++ extractOutputs(simulationOutput.last)
case Some(f) =>
import xml._
def toVariable(v: Val[_], value: Array[String]) =
v match {
case Val.caseArrayInt(v) => Variable(v, value.map(_.toInt))
case Val.caseArrayDouble(v) => Variable(v, value.map(_.toDouble))
case Val.caseArrayString(v) => Variable(v, value)
case Val.caseArrayBoolean(v) => Variable(v, value.map(_.toBoolean))
case _ => throw new UserBadDataError(s"Unsupported type of output variable $v")
}
def outputValue(e: Elem, name: String) =
for {
a <- e.attribute("name").flatMap(_.headOption)
if a.text == name
} yield e.child.text
val simulationOutput = XML.loadFile(tmpOutputDirectory / "simulation-outputs0.xml") \ "Step"
resultContext ++ mapped.outputs.map { m =>
val values =
for {
o <- simulationOutput
v <- o \ "Variable"
} yield
v match {
case o: Elem => outputValue(o, m.name)
case _ => None
}
toVariable(m.v, values.flatten.toArray)
}
}
}
}
}
}
......@@ -33,7 +33,7 @@ object PythonTask {
def installCommands(install: Seq[String], libraries: Seq[String], major: Int): Vector[String] = {
// need to install pip2 in case of python 2
val effintsall = install++(if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty)
val effintsall = install ++ (if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty)
(effintsall ++ libraries.map { l "pip"+major+" install " + l }).toVector
}
......@@ -43,7 +43,6 @@ object PythonTask {
major: Int = 3,
libraries: Seq[String] = Seq.empty,
install: Seq[String] = Seq.empty,
forceUpdate: Boolean = false,
workDirectory: OptionalArgument[String] = None,
hostFiles: Seq[HostFile] = Vector.empty,
environmentVariables: Seq[EnvironmentVariable] = Vector.empty,
......
......@@ -55,7 +55,6 @@ object RTask {
script: RunnableScript,
install: Seq[String] = Seq.empty,
libraries: Seq[InstallCommand] = Seq.empty,
forceUpdate: Boolean = false,
version: String = "3.6.2",
errorOnReturnValue: Boolean = true,
returnValue: OptionalArgument[Val[Int]] = None,
......
......@@ -33,7 +33,6 @@ object ScilabTask {
def apply(
script: RunnableScript,
install: Seq[String] = Seq.empty,
forceUpdate: Boolean = false,
version: String = "6.1.0",
errorOnReturnValue: Boolean = true,
returnValue: OptionalArgument[Val[Int]] = None,
......
......@@ -8,7 +8,7 @@ object Libraries {
lazy val gridscaleVersion = "2.28"
lazy val sshjVersion = "0.27.0"
lazy val containerVersion = "1.9"
lazy val containerVersion = "1.10"
lazy val mgoVersion = "3.43"
lazy val bouncyCastleVersion = "1.64"
lazy val d3Version = "3.5.12"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment