Commit c3e6d3d0 authored by Romain Reuillon's avatar Romain Reuillon
Browse files

Merge branch 'master' into 9-dev

parents 9b8e4869 0a7b31b0
Pipeline #324 failed with stage
in 5 minutes and 42 seconds
package org.openmole.plugin.task.python package org.openmole.plugin.task.python
import org.openmole.core.context.{ Context, Val } import monocle.macros._
import org.openmole.core.context.{Context, Val}
import org.openmole.core.dsl._ import org.openmole.core.dsl._
import org.openmole.core.dsl.extension._ import org.openmole.core.dsl.extension._
import org.openmole.core.fileservice.FileService import org.openmole.core.fileservice.FileService
...@@ -9,129 +10,177 @@ import org.openmole.core.networkservice.NetworkService ...@@ -9,129 +10,177 @@ import org.openmole.core.networkservice.NetworkService
import org.openmole.core.preference.Preference import org.openmole.core.preference.Preference
import org.openmole.core.threadprovider.ThreadProvider import org.openmole.core.threadprovider.ThreadProvider
import org.openmole.core.workflow.builder._ import org.openmole.core.workflow.builder._
import org.openmole.core.workflow.task.TaskExecutionContext
import org.openmole.core.workflow.tools.OptionalArgument import org.openmole.core.workflow.tools.OptionalArgument
import org.openmole.core.workspace.{ NewFile, Workspace } import org.openmole.core.workflow.validation.ValidateTask
import org.openmole.plugin.task.container.HostFile import org.openmole.core.workspace.{NewFile, Workspace}
import org.openmole.plugin.task.container.{HostFile, HostFiles}
import org.openmole.plugin.task.external._ import org.openmole.plugin.task.external._
import org.openmole.plugin.task.udocker._ import org.openmole.plugin.task.udocker._
import org.openmole.plugin.tool.json._ import org.openmole.plugin.tool.json._
import org.openmole.tool.outputredirection.OutputRedirection import org.openmole.tool.outputredirection.OutputRedirection
import org.openmole.plugin.task.container
import org.openmole.plugin.task.systemexec._
object PythonTask { object PythonTask {
// could to make distinct image for python 2 and 3 implicit def isTask: InputOutputBuilder[PythonTask] = InputOutputBuilder(PythonTask.config)
def dockerImage(major: Int) = DockerImage("python") implicit def isExternal: ExternalBuilder[PythonTask] = ExternalBuilder(PythonTask.external)
implicit def isInfo = InfoBuilder(info)
def installCommands(install: Seq[String], libraries: Seq[String], major: Int): Vector[String] = { implicit def isMapped = MappedInputOutputBuilder(PythonTask.mapped)
// need to install pip2 in case of python 2
val effintsall = install++(if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty) implicit def isBuilder = new ReturnValue[PythonTask] with ErrorOnReturnValue[PythonTask] with StdOutErr[PythonTask] with EnvironmentVariables[PythonTask] with HostFiles[PythonTask] with WorkDirectory[PythonTask] { builder
(effintsall ++ libraries.map { l "pip"+major+" install " + l }).toVector override def returnValue = PythonTask.returnValue
override def errorOnReturnValue = PythonTask.errorOnReturnValue
override def stdOut = PythonTask.stdOut
override def stdErr = PythonTask.stdErr
override def environmentVariables = PythonTask.uDocker composeLens UDockerArguments.environmentVariables
override def hostFiles = PythonTask.uDocker composeLens UDockerArguments.hostFiles
override def workDirectory = PythonTask.uDocker composeLens UDockerArguments.workDirectory
} }
// could to make distinct image for python 2 and 3
def dockerImage(major: Int) = DockerImage("python")
def installCommands(install: Seq[String], libraries: Seq[String], major: Int): Vector[String] = {
// need to install pip2 in case of python 2
val effintsall = install++(if (major==2) Seq("curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py","python2 get-pip.py") else Seq.empty)
(effintsall ++ libraries.map { l "pip"+major+" install " + l }).toVector
}
def apply(
script: RunnableScript,
major: Int = 3,
libraries: Seq[String] = Seq.empty,
install: Seq[String] = Seq.empty,
forceUpdate: Boolean = false,
workDirectory: OptionalArgument[String] = None,
hostFiles: Seq[HostFile] = Vector.empty,
environmentVariables: Seq[EnvironmentVariable] = Vector.empty,
errorOnReturnValue: Boolean = true,
returnValue: OptionalArgument[Val[Int]] = None,
stdOut: OptionalArgument[Val[String]] = None,
stdErr: OptionalArgument[Val[String]] = None)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: NewFile, workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService) = {
val uDocker =
UDockerTask.createUDocker(
dockerImage(major),
install = installCommands(install, libraries,major),
cacheInstall = true,
forceUpdate = forceUpdate,
mode = "P1",
reuseContainer = true,
hostFiles = hostFiles,
workDirectory = workDirectory,
environmentVariables = environmentVariables.toVector)
new PythonTask(
script = script,
uDocker,
errorOnReturnValue = errorOnReturnValue,
returnValue = returnValue,
stdOut = stdOut,
stdErr = stdErr,
config = InputOutputConfig(),
external = External(),
info = InfoConfig(),
mapped = MappedInputOutputConfig(),
major = major
) set (outputs += (Seq(returnValue.option, stdOut.option, stdErr.option).flatten: _*))
}
}
def apply( @Lenses case class PythonTask(
script: RunnableScript, script: RunnableScript,
major: Int = 3, uDocker: UDockerArguments,
libraries: Seq[String] = Seq.empty, errorOnReturnValue: Boolean,
install: Seq[String] = Seq.empty, returnValue: Option[Val[Int]],
forceUpdate: Boolean = false, stdOut: Option[Val[String]],
workDirectory: OptionalArgument[String] = None, stdErr: Option[Val[String]],
hostFiles: Seq[HostFile] = Vector.empty, config: InputOutputConfig,
environmentVariables: Seq[EnvironmentVariable] = Vector.empty, external: External,
errorOnReturnValue: Boolean = true, info: InfoConfig,
returnValue: OptionalArgument[Val[Int]] = None, mapped: MappedInputOutputConfig,
stdOut: OptionalArgument[Val[String]] = None, major: Int) extends Task with ValidateTask {
stdErr: OptionalArgument[Val[String]] = None)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: NewFile, workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService) = {
lazy val containerPoolKey = UDockerTask.newCacheKey
val udocker =
UDockerTask.createUDocker(
dockerImage(major),
install = installCommands(install, libraries,major),
cacheInstall = true,
forceUpdate = forceUpdate,
mode = "P1",
reuseContainer = true,
hostFiles = hostFiles,
workDirectory = workDirectory,
environmentVariables = environmentVariables.toVector)
Task("PythonTask") { p
import org.json4s.jackson.JsonMethods._
import p._
import Mapped.noFile
def writeInputsJSON(file: File): Unit = { lazy val containerPoolKey = UDockerTask.newCacheKey
def values = noFile(mapped.inputs).map { m => p.context(m.v) }
file.content = compact(render(toJSONValue(values.toArray)))
}
def readOutputJSON(file: File) = { override def validate = container.validateContainer(Vector(), uDocker.environmentVariables, external, inputs)
import org.json4s._
import org.json4s.jackson.JsonMethods._
val outputValues = parse(file.content)
(outputValues.asInstanceOf[JArray].arr zip noFile(mapped.outputs).map(_.v)).map { case (jvalue, v) jValueToVariable(jvalue, v) }
}
def inputMapping(dicoName: String): String =
noFile(mapped.inputs).zipWithIndex.map { override def process(executionContext: TaskExecutionContext) = FromContext { p
case (m, i) s"${m.name} = $dicoName[${i}]" import org.json4s.jackson.JsonMethods._
}.mkString("\n") import p._
import Mapped.noFile
def outputMapping: String =
s"""[${noFile(mapped.outputs).map { m ⇒ m.name }.mkString(",")}]""" def writeInputsJSON(file: File): Unit = {
def values = noFile(mapped.inputs).map { m => p.context(m.v) }
val resultContext: Context = p.newFile.withTmpFile("script", ".py") { scriptFile file.content = compact(render(toJSONValue(values.toArray)))
p.newFile.withTmpFile("inputs", ".json") { jsonInputs }
def inputArrayName = "_generateddata_" def readOutputJSON(file: File) = {
def scriptName = "_generatescript_.py" import org.json4s._
def inputJSONName = "_inputs_.json" import org.json4s.jackson.JsonMethods._
def outputJSONName = "_outputs_.json" val outputValues = parse(file.content)
(outputValues.asInstanceOf[JArray].arr zip noFile(mapped.outputs).map(_.v)).map { case (jvalue, v) jValueToVariable(jvalue, v) }
writeInputsJSON(jsonInputs) }
scriptFile.content =
s""" def inputMapping(dicoName: String): String =
|import json noFile(mapped.inputs).zipWithIndex.map {
|f = open('/$inputJSONName','r') case (m, i) s"${m.name} = $dicoName[${i}]"
|print(f.readlines()) }.mkString("\n")
|$inputArrayName = json.load(open('/$inputJSONName'))
|${inputMapping(inputArrayName)} def outputMapping: String =
|${RunnableScript.content(script)} s"""[${noFile(mapped.outputs).map { m ⇒ m.name }.mkString(",")}]"""
|json.dump($outputMapping, open('/$outputJSONName','w'))
""".stripMargin val resultContext: Context = p.newFile.withTmpFile("script", ".py") { scriptFile
p.newFile.withTmpFile("inputs", ".json") { jsonInputs
val outputFile = Val[File]("outputFile", Namespace("PythonTask"))
def inputArrayName = "_generateddata_"
def uDockerTask = def scriptName = "_generatescript_.py"
UDockerTask( def inputJSONName = "_inputs_.json"
udocker, def outputJSONName = "_outputs_.json"
commands = s"python${major.toString} $scriptName",
errorOnReturnValue = errorOnReturnValue, writeInputsJSON(jsonInputs)
returnValue = returnValue, scriptFile.content =
stdOut = stdOut, s"""
stdErr = stdErr, |import json
config = InputOutputConfig(), |f = open('/$inputJSONName','r')
external = External(), |print(f.readlines())
info = InfoConfig(), |$inputArrayName = json.load(open('/$inputJSONName'))
containerPoolKey = containerPoolKey) set ( |${inputMapping(inputArrayName)}
resources += (scriptFile, scriptName, true), |${RunnableScript.content(script)}
resources += (jsonInputs, inputJSONName, true), |json.dump($outputMapping, open('/$outputJSONName','w'))
outputFiles += (outputJSONName, outputFile), """.stripMargin
Mapped.files(mapped.inputs).map { case m inputFiles +=[UDockerTask] (m.v, m.name, true) },
Mapped.files(mapped.outputs).map { case m outputFiles +=[UDockerTask] (m.name, m.v) } val outputFile = Val[File]("outputFile", Namespace("PythonTask"))
)
def uDockerTask =
val resultContext = uDockerTask.process(p.executionContext).from(p.context)(p.random, p.newFile, p.fileService) UDockerTask(
resultContext ++ readOutputJSON(resultContext(outputFile)) uDocker,
} commands = s"python${major.toString} $scriptName",
errorOnReturnValue = errorOnReturnValue,
returnValue = returnValue,
stdOut = stdOut,
stdErr = stdErr,
config = InputOutputConfig(),
external = external,
info = info,
containerPoolKey = containerPoolKey) set (
resources += (scriptFile, scriptName, true),
resources += (jsonInputs, inputJSONName, true),
outputFiles += (outputJSONName, outputFile),
Mapped.files(mapped.inputs).map { case m inputFiles +=[UDockerTask] (m.v, m.name, true) },
Mapped.files(mapped.outputs).map { case m outputFiles +=[UDockerTask] (m.name, m.v) }
)
val resultContext = uDockerTask.process(executionContext).from(p.context)(p.random, p.newFile, p.fileService)
resultContext ++ readOutputJSON(resultContext(outputFile))
} }
resultContext }
} validate { _ Seq.empty } set ( resultContext
outputs += (Seq(returnValue.option, stdOut.option, stdErr.option).flatten: _*)
)
} }
} }
...@@ -123,7 +123,7 @@ object RTask { ...@@ -123,7 +123,7 @@ object RTask {
returnValue: Option[Val[Int]], returnValue: Option[Val[Int]],
stdOut: Option[Val[String]], stdOut: Option[Val[String]],
stdErr: Option[Val[String]], stdErr: Option[Val[String]],
config: InputOutputConfig, config: InputOutputConfig,
external: External, external: External,
info: InfoConfig, info: InfoConfig,
mapped: MappedInputOutputConfig) extends Task with ValidateTask { mapped: MappedInputOutputConfig) extends Task with ValidateTask {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment