Commit 88ac0739 authored by Romain Reuillon's avatar Romain Reuillon
Browse files

[Plugin] fix: reimplement command line parsing (because we can and also...

[Plugin] fix: reimplement command line parsing (because we can and also because apache exec is buggy).
parent ea322c05
......@@ -259,7 +259,7 @@ lazy val math = OsgiProject(dir, "org.apache.commons.math", exports = Seq("org.a
(libraryDependencies += "org.apache.commons" % "commons-math3" % mathVersion, version := mathVersion) settings(settings: _*)
lazy val exec = OsgiProject(dir, "org.apache.commons.exec") settings
(libraryDependencies += "org.apache.commons" % "commons-exec" % "1.1", version := "1.1") settings(settings: _*)
(libraryDependencies += "org.apache.commons" % "commons-exec" % "1.3", version := "1.3") settings(settings: _*)
lazy val log4j = OsgiProject(dir, "org.apache.log4j") settings
(libraryDependencies += "log4j" % "log4j" % "1.2.17", version := "1.2.17") settings(settings: _*)
......
......@@ -171,14 +171,14 @@ object CARETask extends Logger {
reExecute.setExecutable(true)
val commandline = commandLine(command.map(s"./${reExecute.getName} " + _), userWorkDirectory).from(preparedContext)
val cl = commandLine(command.map(s"./${reExecute.getName} " + _), userWorkDirectory).from(preparedContext)
def prootNoSeccomp = if (preference(CARETask.disableSeccomp)) Vector(("PROOT_NO_SECCOMP", "1")) else Vector()
val allEnvironmentVariables = environmentVariables.map { case (varName, variable) (varName, variable.from(context)) } ++ prootNoSeccomp
val executionResult = execute(commandline, extractedArchive, allEnvironmentVariables, stdOut.isDefined, stdErr.isDefined)
val executionResult = execute(cl.toArray, extractedArchive, allEnvironmentVariables, stdOut.isDefined, stdErr.isDefined)
if (errorOnReturnValue && returnValue.isEmpty && executionResult.returnCode != 0) throw error(commandline.toVector, executionResult)
if (errorOnReturnValue && returnValue.isEmpty && executionResult.returnCode != 0) throw error(cl, executionResult)
def rootDirectory = extractedArchive / rootfs
......
......@@ -17,7 +17,7 @@ import org.openmole.core.dsl._
object RTask {
def installLibraries(libraries: Seq[String]) = s"""R -e "install.packages(c(${libraries.map(lib ⇒ s"'$lib'").mkString(",")}), dependencies = T)""""
def installLibraries(libraries: Seq[String]) = s"""R -e 'install.packages(c(${libraries.map(lib ⇒ '"' + s"$lib" + '"').mkString(",")}), dependencies = T)'"""
def rImage(version: OptionalArgument[String]) = DockerImage("r-base", version.getOrElse("latest"))
def apply(
......@@ -28,7 +28,7 @@ object RTask {
)(implicit name: sourcecode.Name, newFile: NewFile, workspace: Workspace, preference: Preference, fileService: FileService) =
UDockerTask(
rImage(version),
s"R --slave -f ${script.getName}" + arguments.map(a " --args ${a}").getOrElse(""),
s"R --slave -f ${script.getName}" + arguments.map(a s" --args ${a}").getOrElse(""),
installCommands = Vector(installLibraries(libraries))
) set (
resources += script,
......
......@@ -18,9 +18,9 @@
package org.openmole.plugin.task
import java.io.{ File, IOException, PrintStream }
import java.text.ParseException
import monocle.Lens
import org.apache.commons.exec.CommandLine
import org.openmole.core.context._
import org.openmole.core.exception._
import org.openmole.core.expansion.FromContext
......@@ -176,14 +176,14 @@ package object systemexec extends external.ExternalPackage with SystemExecPackag
case class ExecutionResult(returnCode: Int, output: Option[String], errorOutput: Option[String])
def commandLine(cmd: String) = CommandLine.parse(cmd).toStrings
def commandLine(cmd: String) = parse(cmd)
def commandLine(
cmd: FromContext[String],
workDir: String
) = FromContext { p
import p._
CommandLine.parse(cmd.from(context + Variable(External.PWD, workDir))).toStrings
parse(cmd.from(context + Variable(External.PWD, workDir)))
}
def execute(
......@@ -269,11 +269,63 @@ package object systemexec extends external.ExternalPackage with SystemExecPackag
cmds match {
case Nil acc
case cmd :: t
val commandline = commandLine(cmd, workDirectory.getAbsolutePath).from(context)
val result = execute(commandline, workDirectory, environmentVariables, returnOutput = stdOut.isDefined, returnError = stdErr.isDefined, errorOnReturnValue = false)
if (errorOnReturnValue && !returnValue.isDefined && result.returnCode != 0) throw error(commandline.toVector, result)
val cl = commandLine(cmd, workDirectory.getAbsolutePath).from(context)
val result = execute(cl.toArray, workDirectory, environmentVariables, returnOutput = stdOut.isDefined, returnError = stdErr.isDefined, errorOnReturnValue = false)
if (errorOnReturnValue && !returnValue.isDefined && result.returnCode != 0) throw error(cl, result)
else executeAll(workDirectory, environmentVariables, errorOnReturnValue, returnValue, stdOut, stdErr, t, ExecutionResult.append(acc, result))(p)
}
}
def parse(line: String) = {
var inDoubleQuote = false
var inSingleQuote = false
var blocked = false
var afterSpace = false
val arguments = collection.mutable.ListBuffer[String]()
val currentArguments = new StringBuilder
def addArgument() = {
arguments += currentArguments.toString()
currentArguments.clear()
}
for (character line) {
(inDoubleQuote, inSingleQuote, blocked, character, afterSpace) match {
case (_, _, true, c, _)
currentArguments.append(c)
blocked = false
case (_, _, false, '\\', _) blocked = true
case (false, false, false, ' ', _)
if (!currentArguments.isEmpty) addArgument()
case (false, false, false, '"', _)
inDoubleQuote = true
// currentArguments.append(character)
case (true, false, false, '"', false)
inDoubleQuote = false
//currentArguments.append(character)
case (false, false, false, ''', _)
inSingleQuote = true
//currentArguments.append(character)
case (false, true, false, ''', false)
inSingleQuote = false
//currentArguments.append(character)
case (false, false, false, c, _) currentArguments.append(c)
case (true, false, false, c, _) currentArguments.append(c)
case (false, true, false, c, _) currentArguments.append(c)
case _ throw new RuntimeException(s"Error while parsing command line: $line")
}
afterSpace = character == ' '
}
if (!currentArguments.isEmpty) addArgument()
arguments.toVector
}
}
package org.openmole.plugin.task.systemexec
import org.scalatest._
class ParserSpec extends FlatSpec with Matchers {
"Command line arguments" should "be parsed propely" in {
val l1 = """R -e "install.packages(c(\"lib\"), dependencies = T)""""
parse(l1).size should equal(3)
val l2 = s"/tmp/udocker create --name=eiav80eaiuE imageId"
parse(l2).size should equal(4)
val l3 = """/home/reuillon/.openmole/simplet/.tmp/3d48df46-a2b5-46f3-bcf8-e0a99bb0f2de/execution9ee99d7e-52e4-4ff4-bd6a-6942b7104eec/udocker/udocker run --workdir="/" -v "/home/reuillon/.openmole/simplet/.tmp/3d48df46-a2b5-46f3-bcf8-e0a99bb0f2de/execution9ee99d7e-52e4-4ff4-bd6a-6942b7104eec/externalTask035caba5-a447-4267-8f5d-a67e5fdc80b6/inputs/data.csv":"/data.csv" -v "/home/reuillon/.openmole/simplet/.tmp/3d48df46-a2b5-46f3-bcf8-e0a99bb0f2de/execution9ee99d7e-52e4-4ff4-bd6a-6942b7104eec/externalTask035caba5-a447-4267-8f5d-a67e5fdc80b6/inputs/test.R":"/test.R" jfdoimamfhkdkbekapobfgofgdebbjni R -e "install.packages(c('spdep'), dependencies = T)""""
parse(l3).size should equal(11)
}
}
......@@ -316,11 +316,15 @@ object UDockerTask {
def newContainer(imageId: String)() = newFile.withTmpDir { tmpDirectory
val name = containerName(UUID.randomUUID().toString)
val commandline = commandLine(s"${uDockerFile.getAbsolutePath} create --name=$name $imageId")
execute(commandline, tmpDirectory, udockerVariables(), returnOutput = true, returnError = true)
val cl = commandLine(s"${uDockerFile.getAbsolutePath} create --name=$name $imageId")
execute(cl.toArray, tmpDirectory, udockerVariables(), returnOutput = true, returnError = true)
if (!uDocker.installCommands.isEmpty) {
val runInstall = uDocker.installCommands.map(ic runCommand(uDocker)(uDockerFile, volumes.toVector, name, ic))
println(runInstall.map(_.from(preparedContext)))
executeAll(
tmpDirectory,
udockerVariables(),
......
......@@ -58,7 +58,7 @@ object Libraries {
lazy val scalatexSite = "org.openmole.library" %% "com-lihaoyi-scalatex-site" % "0.3.7"
lazy val math = "org.openmole.library" %% "org-apache-commons-math" % "3.6.1"
lazy val collections = "org.openmole.library" %% "org-apache-commons-collections" % "4.1"
lazy val exec = "org.openmole.library" %% "org-apache-commons-exec" % "1.1"
lazy val exec = "org.openmole.library" %% "org-apache-commons-exec" % "1.3"
lazy val log4j = "org.openmole.library" %% "org-apache-log4j" % "1.2.17"
lazy val logging = "org.openmole.library" %% "org-apache-commons-logging" % "1.2"
lazy val lang3 = "org.openmole.library" %% "org-apache-commons-lang3" % "3.4"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment