Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
openmole
openmole
Commits
0a7b31b0
Commit
0a7b31b0
authored
Jun 26, 2019
by
Romain Reuillon
Browse files
[Plugin] fix: enable resources keyword for PythonTask
parent
4fea990c
Changes
2
Hide whitespace changes
Inline
Side-by-side
openmole/plugins/org.openmole.plugin.task.python/src/main/scala/org/openmole/plugin/task/python/PythonTask.scala
View file @
0a7b31b0
package
org.openmole.plugin.task.python
import
org.openmole.core.context.
{
Context
,
Val
}
import
monocle.macros._
import
org.openmole.core.context.
{
Context
,
Val
}
import
org.openmole.core.dsl._
import
org.openmole.core.dsl.extension._
import
org.openmole.core.fileservice.FileService
...
...
@@ -9,129 +10,177 @@ import org.openmole.core.networkservice.NetworkService
import
org.openmole.core.preference.Preference
import
org.openmole.core.threadprovider.ThreadProvider
import
org.openmole.core.workflow.builder._
import
org.openmole.core.workflow.task.TaskExecutionContext
import
org.openmole.core.workflow.tools.OptionalArgument
import
org.openmole.core.workspace.
{
NewFile
,
Workspace
}
import
org.openmole.plugin.task.container.HostFile
import
org.openmole.core.workflow.validation.ValidateTask
import
org.openmole.core.workspace.
{
NewFile
,
Workspace
}
import
org.openmole.plugin.task.container.
{
HostFile
,
HostFiles
}
import
org.openmole.plugin.task.external._
import
org.openmole.plugin.task.udocker._
import
org.openmole.plugin.tool.json._
import
org.openmole.tool.outputredirection.OutputRedirection
import
org.openmole.plugin.task.container
import
org.openmole.plugin.task.systemexec._
object
PythonTask
{
// could to make distinct image for python 2 and 3
def
dockerImage
(
major
:
Int
)
=
DockerImage
(
"python"
)
def
installCommands
(
install
:
Seq
[
String
],
libraries
:
Seq
[
String
],
major
:
Int
)
:
Vector
[
String
]
=
{
// need to install pip2 in case of python 2
val
effintsall
=
install
++(
if
(
major
==
2
)
Seq
(
"curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
,
"python2 get-pip.py"
)
else
Seq
.
empty
)
(
effintsall
++
libraries
.
map
{
l
⇒
"pip"
+
major
+
" install "
+
l
}).
toVector
implicit
def
isTask
:
InputOutputBuilder
[
PythonTask
]
=
InputOutputBuilder
(
PythonTask
.
config
)
implicit
def
isExternal
:
ExternalBuilder
[
PythonTask
]
=
ExternalBuilder
(
PythonTask
.
external
)
implicit
def
isInfo
=
InfoBuilder
(
info
)
implicit
def
isMapped
=
MappedInputOutputBuilder
(
PythonTask
.
mapped
)
implicit
def
isBuilder
=
new
ReturnValue
[
PythonTask
]
with
ErrorOnReturnValue
[
PythonTask
]
with
StdOutErr
[
PythonTask
]
with
EnvironmentVariables
[
PythonTask
]
with
HostFiles
[
PythonTask
]
with
WorkDirectory
[
PythonTask
]
{
builder
⇒
override
def
returnValue
=
PythonTask
.
returnValue
override
def
errorOnReturnValue
=
PythonTask
.
errorOnReturnValue
override
def
stdOut
=
PythonTask
.
stdOut
override
def
stdErr
=
PythonTask
.
stdErr
override
def
environmentVariables
=
PythonTask
.
uDocker
composeLens
UDockerArguments
.
environmentVariables
override
def
hostFiles
=
PythonTask
.
uDocker
composeLens
UDockerArguments
.
hostFiles
override
def
workDirectory
=
PythonTask
.
uDocker
composeLens
UDockerArguments
.
workDirectory
}
// could to make distinct image for python 2 and 3
def
dockerImage
(
major
:
Int
)
=
DockerImage
(
"python"
)
def
installCommands
(
install
:
Seq
[
String
],
libraries
:
Seq
[
String
],
major
:
Int
)
:
Vector
[
String
]
=
{
// need to install pip2 in case of python 2
val
effintsall
=
install
++(
if
(
major
==
2
)
Seq
(
"curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
,
"python2 get-pip.py"
)
else
Seq
.
empty
)
(
effintsall
++
libraries
.
map
{
l
⇒
"pip"
+
major
+
" install "
+
l
}).
toVector
}
def
apply
(
script
:
RunnableScript
,
major
:
Int
=
3
,
libraries
:
Seq
[
String
]
=
Seq
.
empty
,
install
:
Seq
[
String
]
=
Seq
.
empty
,
forceUpdate
:
Boolean
=
false
,
workDirectory
:
OptionalArgument
[
String
]
=
None
,
hostFiles
:
Seq
[
HostFile
]
=
Vector
.
empty
,
environmentVariables
:
Seq
[
EnvironmentVariable
]
=
Vector
.
empty
,
errorOnReturnValue
:
Boolean
=
true
,
returnValue
:
OptionalArgument
[
Val
[
Int
]]
=
None
,
stdOut
:
OptionalArgument
[
Val
[
String
]]
=
None
,
stdErr
:
OptionalArgument
[
Val
[
String
]]
=
None
)(
implicit
name
:
sourcecode.Name
,
definitionScope
:
DefinitionScope
,
newFile
:
NewFile
,
workspace
:
Workspace
,
preference
:
Preference
,
fileService
:
FileService
,
threadProvider
:
ThreadProvider
,
outputRedirection
:
OutputRedirection
,
networkService
:
NetworkService
)
=
{
val
uDocker
=
UDockerTask
.
createUDocker
(
dockerImage
(
major
),
install
=
installCommands
(
install
,
libraries
,
major
),
cacheInstall
=
true
,
forceUpdate
=
forceUpdate
,
mode
=
"P1"
,
reuseContainer
=
true
,
hostFiles
=
hostFiles
,
workDirectory
=
workDirectory
,
environmentVariables
=
environmentVariables
.
toVector
)
new
PythonTask
(
script
=
script
,
uDocker
,
errorOnReturnValue
=
errorOnReturnValue
,
returnValue
=
returnValue
,
stdOut
=
stdOut
,
stdErr
=
stdErr
,
config
=
InputOutputConfig
(),
external
=
External
(),
info
=
InfoConfig
(),
mapped
=
MappedInputOutputConfig
(),
major
=
major
)
set
(
outputs
+=
(
Seq
(
returnValue
.
option
,
stdOut
.
option
,
stdErr
.
option
).
flatten
:
_
*
))
}
}
def
apply
(
script
:
RunnableScript
,
major
:
Int
=
3
,
libraries
:
Seq
[
String
]
=
Seq
.
empty
,
install
:
Seq
[
String
]
=
Seq
.
empty
,
forceUpdate
:
Boolean
=
false
,
workDirectory
:
OptionalArgument
[
String
]
=
None
,
hostFiles
:
Seq
[
HostFile
]
=
Vector
.
empty
,
environmentVariables
:
Seq
[
EnvironmentVariable
]
=
Vector
.
empty
,
errorOnReturnValue
:
Boolean
=
true
,
returnValue
:
OptionalArgument
[
Val
[
Int
]]
=
None
,
stdOut
:
OptionalArgument
[
Val
[
String
]]
=
None
,
stdErr
:
OptionalArgument
[
Val
[
String
]]
=
None
)(
implicit
name
:
sourcecode.Name
,
definitionScope
:
DefinitionScope
,
newFile
:
NewFile
,
workspace
:
Workspace
,
preference
:
Preference
,
fileService
:
FileService
,
threadProvider
:
ThreadProvider
,
outputRedirection
:
OutputRedirection
,
networkService
:
NetworkService
)
=
{
lazy
val
containerPoolKey
=
UDockerTask
.
newCacheKey
val
udocker
=
UDockerTask
.
createUDocker
(
dockerImage
(
major
),
install
=
installCommands
(
install
,
libraries
,
major
),
cacheInstall
=
true
,
forceUpdate
=
forceUpdate
,
mode
=
"P1"
,
reuseContainer
=
true
,
hostFiles
=
hostFiles
,
workDirectory
=
workDirectory
,
environmentVariables
=
environmentVariables
.
toVector
)
Task
(
"PythonTask"
)
{
p
⇒
import
org.json4s.jackson.JsonMethods._
import
p._
import
Mapped.noFile
@Lenses
case
class
PythonTask
(
script
:
RunnableScript
,
uDocker
:
UDockerArguments
,
errorOnReturnValue
:
Boolean
,
returnValue
:
Option
[
Val
[
Int
]],
stdOut
:
Option
[
Val
[
String
]],
stdErr
:
Option
[
Val
[
String
]],
config
:
InputOutputConfig
,
external
:
External
,
info
:
InfoConfig
,
mapped
:
MappedInputOutputConfig
,
major
:
Int
)
extends
Task
with
ValidateTask
{
def
writeInputsJSON
(
file
:
File
)
:
Unit
=
{
def
values
=
noFile
(
mapped
.
inputs
).
map
{
m
=>
p
.
context
(
m
.
v
)
}
file
.
content
=
compact
(
render
(
toJSONValue
(
values
.
toArray
)))
}
lazy
val
containerPoolKey
=
UDockerTask
.
newCacheKey
def
readOutputJSON
(
file
:
File
)
=
{
import
org.json4s._
import
org.json4s.jackson.JsonMethods._
val
outputValues
=
parse
(
file
.
content
)
(
outputValues
.
asInstanceOf
[
JArray
].
arr
zip
noFile
(
mapped
.
outputs
).
map
(
_
.
v
)).
map
{
case
(
jvalue
,
v
)
⇒
jValueToVariable
(
jvalue
,
v
)
}
}
override
def
validate
=
container
.
validateContainer
(
Vector
(),
uDocker
.
environmentVariables
,
external
,
inputs
)
def
inputMapping
(
dicoName
:
String
)
:
String
=
noFile
(
mapped
.
inputs
).
zipWithIndex
.
map
{
case
(
m
,
i
)
⇒
s
"${m.name} = $dicoName[${i}]"
}.
mkString
(
"\n"
)
def
outputMapping
:
String
=
s
"""[${noFile(mapped.outputs).map { m ⇒ m.name }.mkString(",")}]"""
val
resultContext
:
Context
=
p
.
newFile
.
withTmpFile
(
"script"
,
".py"
)
{
scriptFile
⇒
p
.
newFile
.
withTmpFile
(
"inputs"
,
".json"
)
{
jsonInputs
⇒
def
inputArrayName
=
"_generateddata_"
def
scriptName
=
"_generatescript_.py"
def
inputJSONName
=
"_inputs_.json"
def
outputJSONName
=
"_outputs_.json"
writeInputsJSON
(
jsonInputs
)
scriptFile
.
content
=
s
"""
|import json
|f = open('/$inputJSONName','r')
|print(f.readlines())
|$inputArrayName = json.load(open('/$inputJSONName'))
|${inputMapping(inputArrayName)}
|${RunnableScript.content(script)}
|json.dump($outputMapping, open('/$outputJSONName','w'))
"""
.
stripMargin
val
outputFile
=
Val
[
File
](
"outputFile"
,
Namespace
(
"PythonTask"
))
def
uDockerTask
=
UDockerTask
(
udocker
,
commands
=
s
"python${major.toString} $scriptName"
,
errorOnReturnValue
=
errorOnReturnValue
,
returnValue
=
returnValue
,
stdOut
=
stdOut
,
stdErr
=
stdErr
,
config
=
InputOutputConfig
(),
external
=
External
(),
info
=
InfoConfig
(),
containerPoolKey
=
containerPoolKey
)
set
(
resources
+=
(
scriptFile
,
scriptName
,
true
),
resources
+=
(
jsonInputs
,
inputJSONName
,
true
),
outputFiles
+=
(
outputJSONName
,
outputFile
),
Mapped
.
files
(
mapped
.
inputs
).
map
{
case
m
⇒
inputFiles
+=[
UDockerTask
]
(
m
.
v
,
m
.
name
,
true
)
},
Mapped
.
files
(
mapped
.
outputs
).
map
{
case
m
⇒
outputFiles
+=[
UDockerTask
]
(
m
.
name
,
m
.
v
)
}
)
val
resultContext
=
uDockerTask
.
process
(
p
.
executionContext
).
from
(
p
.
context
)(
p
.
random
,
p
.
newFile
,
p
.
fileService
)
resultContext
++
readOutputJSON
(
resultContext
(
outputFile
))
}
override
def
process
(
executionContext
:
TaskExecutionContext
)
=
FromContext
{
p
⇒
import
org.json4s.jackson.JsonMethods._
import
p._
import
Mapped.noFile
def
writeInputsJSON
(
file
:
File
)
:
Unit
=
{
def
values
=
noFile
(
mapped
.
inputs
).
map
{
m
=>
p
.
context
(
m
.
v
)
}
file
.
content
=
compact
(
render
(
toJSONValue
(
values
.
toArray
)))
}
def
readOutputJSON
(
file
:
File
)
=
{
import
org.json4s._
import
org.json4s.jackson.JsonMethods._
val
outputValues
=
parse
(
file
.
content
)
(
outputValues
.
asInstanceOf
[
JArray
].
arr
zip
noFile
(
mapped
.
outputs
).
map
(
_
.
v
)).
map
{
case
(
jvalue
,
v
)
⇒
jValueToVariable
(
jvalue
,
v
)
}
}
def
inputMapping
(
dicoName
:
String
)
:
String
=
noFile
(
mapped
.
inputs
).
zipWithIndex
.
map
{
case
(
m
,
i
)
⇒
s
"${m.name} = $dicoName[${i}]"
}.
mkString
(
"\n"
)
def
outputMapping
:
String
=
s
"""[${noFile(mapped.outputs).map { m ⇒ m.name }.mkString(",")}]"""
val
resultContext
:
Context
=
p
.
newFile
.
withTmpFile
(
"script"
,
".py"
)
{
scriptFile
⇒
p
.
newFile
.
withTmpFile
(
"inputs"
,
".json"
)
{
jsonInputs
⇒
def
inputArrayName
=
"_generateddata_"
def
scriptName
=
"_generatescript_.py"
def
inputJSONName
=
"_inputs_.json"
def
outputJSONName
=
"_outputs_.json"
writeInputsJSON
(
jsonInputs
)
scriptFile
.
content
=
s
"""
|import json
|f = open('/$inputJSONName','r')
|print(f.readlines())
|$inputArrayName = json.load(open('/$inputJSONName'))
|${inputMapping(inputArrayName)}
|${RunnableScript.content(script)}
|json.dump($outputMapping, open('/$outputJSONName','w'))
"""
.
stripMargin
val
outputFile
=
Val
[
File
](
"outputFile"
,
Namespace
(
"PythonTask"
))
def
uDockerTask
=
UDockerTask
(
uDocker
,
commands
=
s
"python${major.toString} $scriptName"
,
errorOnReturnValue
=
errorOnReturnValue
,
returnValue
=
returnValue
,
stdOut
=
stdOut
,
stdErr
=
stdErr
,
config
=
InputOutputConfig
(),
external
=
external
,
info
=
info
,
containerPoolKey
=
containerPoolKey
)
set
(
resources
+=
(
scriptFile
,
scriptName
,
true
),
resources
+=
(
jsonInputs
,
inputJSONName
,
true
),
outputFiles
+=
(
outputJSONName
,
outputFile
),
Mapped
.
files
(
mapped
.
inputs
).
map
{
case
m
⇒
inputFiles
+=[
UDockerTask
]
(
m
.
v
,
m
.
name
,
true
)
},
Mapped
.
files
(
mapped
.
outputs
).
map
{
case
m
⇒
outputFiles
+=[
UDockerTask
]
(
m
.
name
,
m
.
v
)
}
)
val
resultContext
=
uDockerTask
.
process
(
executionContext
).
from
(
p
.
context
)(
p
.
random
,
p
.
newFile
,
p
.
fileService
)
resultContext
++
readOutputJSON
(
resultContext
(
outputFile
))
}
resultContext
}
validate
{
_
⇒
Seq
.
empty
}
set
(
outputs
+=
(
Seq
(
returnValue
.
option
,
stdOut
.
option
,
stdErr
.
option
).
flatten
:
_
*
)
)
}
resultContext
}
}
openmole/plugins/org.openmole.plugin.task.r/src/main/scala/org/openmole/plugin/task/r/RTask.scala
View file @
0a7b31b0
...
...
@@ -123,7 +123,7 @@ object RTask {
returnValue
:
Option
[
Val
[
Int
]],
stdOut
:
Option
[
Val
[
String
]],
stdErr
:
Option
[
Val
[
String
]],
config
:
InputOutputConfig
,
config
:
InputOutputConfig
,
external
:
External
,
info
:
InfoConfig
,
mapped
:
MappedInputOutputConfig
)
extends
Task
with
ValidateTask
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment