Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
openmole
openmole
Commits
d18713b2
Unverified
Commit
d18713b2
authored
Mar 30, 2017
by
Romain Reuillon
Browse files
[All] enh: break services in small unit an pass them in parameter all along the stacks.
parent
82aa2c1e
Changes
283
Hide whitespace changes
Inline
Side-by-side
openmole/bin/openmole-naked/src/main/resources/openmole
View file @
d18713b2
...
...
@@ -42,18 +42,19 @@ case "$JVMVERSION" in
*
1
\.
7
\.
*
)
FLAG
=
"
$FLAG
-XX:MaxPermSize=128M"
;;
esac
(
nohup
sh
$LOCATION
/dbserver/bin/openmole-dbserver
>
/dev/null &
)
2>/dev/null
ARGS
=()
while
[[
$#
-gt
0
]]
;
do
case
"
$1
"
in
--
)
ARGS+
=(
"
$@
"
)
;
break
;;
--mem
)
shift
;
MEM
=
$1
;;
--workspace
)
ARGS+
=(
"
$1
"
)
;
shift
;
WORKSPACE
=
$1
;
ARGS+
=(
"
$1
"
)
;;
*
)
ARGS+
=(
"
$1
"
)
;;
esac
shift
done
(
nohup
sh
$LOCATION
/dbserver/bin/openmole-dbserver
$WORKSPACE
>
/dev/null &
)
2>/dev/null
SCRIPT_ARGS
=()
# Try to increase the max number of opened files
...
...
openmole/bin/org.openmole.console/src/main/scala/org/openmole/console/Command.scala
View file @
d18713b2
...
...
@@ -32,7 +32,13 @@ import org.openmole.core.workflow.puzzle._
import
org.openmole.core.workflow.validation.Validation
import
org.openmole.core.module
import
org.openmole.core.pluginmanager.PluginManager
import
org.openmole.core.workspace.Workspace
import
org.openmole.core.preference.Preference
import
org.openmole.core.replication.ReplicaCatalog
import
org.openmole.core.threadprovider.ThreadProvider
import
org.openmole.core.workspace.
{
NewFile
,
Workspace
}
import
org.openmole.core.services._
import
org.openmole.tool.crypto.Cypher
import
org.openmole.tool.random.
{
RandomProvider
,
Seeder
}
class
Command
(
val
console
:
ScalaREPL
,
val
variables
:
ConsoleVariables
)
{
commands
⇒
...
...
@@ -87,13 +93,13 @@ class Command(val console: ScalaREPL, val variables: ConsoleVariables) { command
def
verify
(
mole
:
Mole
)
:
Unit
=
Validation
(
mole
).
foreach
(
println
)
def
encrypted
:
String
=
encrypt
(
Console
.
askPassword
())
def
encrypted
(
implicit
cypher
:
Cypher
)
:
String
=
encrypt
(
Console
.
askPassword
())
def
version
()
=
println
(
s
"""You are running OpenMOLE ${buildinfo.version} - ${buildinfo.name}
|built on the ${buildinfo.version.generationDate}."""
.
stripMargin
)
def
loadAny
(
file
:
File
,
args
:
Seq
[
String
]
=
Seq
.
empty
)
:
AnyRef
=
def
loadAny
(
file
:
File
,
args
:
Seq
[
String
]
=
Seq
.
empty
)
(
implicit
services
:
Services
)
:
AnyRef
=
try
{
val
project
=
new
Project
(
...
...
@@ -111,17 +117,17 @@ class Command(val console: ScalaREPL, val variables: ConsoleVariables) { command
}
finally
ConsoleVariables
.
bindVariables
(
console
,
variables
)
def
load
(
file
:
File
,
args
:
Seq
[
String
]
=
Seq
.
empty
)
:
Puzzle
=
def
load
(
file
:
File
,
args
:
Seq
[
String
]
=
Seq
.
empty
)
(
implicit
services
:
Services
)
:
Puzzle
=
loadAny
(
file
)
match
{
case
res
:
Puzzle
⇒
res
case
x
⇒
throw
new
UserBadDataError
(
"The result is not a puzzle"
)
}
def
modules
(
urls
:
Seq
[
String
]
=
module
.
indexes
)
:
Unit
=
{
def
modules
(
urls
:
OptionalArgument
[
Seq
[
String
]
]
=
None
)(
implicit
preference
:
Preference
,
randomProvider
:
RandomProvider
,
newFile
:
NewFile
)
:
Unit
=
{
val
installedBundles
=
PluginManager
.
bundleHashes
.
map
(
_
.
toString
).
toSet
def
installed
(
components
:
Seq
[
String
])
=
(
components
.
toSet
--
installedBundles
).
isEmpty
urls
.
flatMap
{
urls
.
getOrElse
(
module
.
indexes
).
flatMap
{
url
⇒
module
.
modules
(
url
).
map
{
m
⇒
...
...
@@ -131,9 +137,9 @@ class Command(val console: ScalaREPL, val variables: ConsoleVariables) { command
}.
sorted
.
foreach
(
println
)
}
def
install
(
name
:
String*
)
:
Unit
=
install
(
name
)
def
install
(
names
:
Seq
[
String
],
urls
:
Seq
[
String
]
=
module
.
indexes
)
:
Unit
=
{
val
toInstall
=
urls
.
flatMap
(
url
⇒
module
.
selectableModules
(
url
)).
filter
(
sm
⇒
names
.
contains
(
sm
.
module
.
name
))
def
install
(
name
:
String*
)
(
implicit
preference
:
Preference
,
randomProvider
:
RandomProvider
,
newFile
:
NewFile
,
workspace
:
Workspace
)
:
Unit
=
install
(
name
)
def
install
(
names
:
Seq
[
String
],
urls
:
OptionalArgument
[
Seq
[
String
]
]
=
None
)(
implicit
preference
:
Preference
,
randomProvider
:
RandomProvider
,
newFile
:
NewFile
,
workspace
:
Workspace
)
:
Unit
=
{
val
toInstall
=
urls
.
getOrElse
(
module
.
indexes
).
flatMap
(
url
⇒
module
.
selectableModules
(
url
)).
filter
(
sm
⇒
names
.
contains
(
sm
.
module
.
name
))
if
(
toInstall
.
isEmpty
)
println
(
"The module(s) is/are already installed."
)
else
Console
.
dealWithLoadError
(
module
.
install
(
toInstall
),
interactive
=
true
)
match
{
...
...
openmole/bin/org.openmole.console/src/main/scala/org/openmole/console/Console.scala
View file @
d18713b2
...
...
@@ -20,11 +20,16 @@ package org.openmole.console
import
jline.console.ConsoleReader
import
org.openmole.core.console.ScalaREPL
import
org.openmole.core.exception.UserBadDataError
import
org.openmole.core.preference.Preference
import
org.openmole.core.project._
import
org.openmole.core.replication.ReplicaCatalog
import
org.openmole.core.threadprovider.ThreadProvider
import
org.openmole.core.tools.io.Prettifier._
import
org.openmole.core.workspace._
import
org.openmole.tool.crypto.Cypher
import
org.openmole.tool.file._
import
org.openmole.tool.logger.Logger
import
org.openmole.core.services._
import
scala.annotation.tailrec
import
scala.util._
...
...
@@ -47,29 +52,26 @@ object Console extends Logger {
else
password
}
def
setPassword
(
password
:
String
)
=
try
{
Workspace
.
setPassword
(
password
)
true
}
catch
{
case
e
:
UserBadDataError
⇒
println
(
"
Password
incorrect.
"
)
false
}
def
testPassword
(
password
:
String
)(
implicit
preference
:
Preference
)
:
Boolean
=
{
val
cypher
=
Cypher
(
password
)
Preference
.
passwordIsCorrect
(
cypher
,
preference
)
}
@tailrec
def
initPassword
:
Unit
=
{
if
(
Workspace
.
passwordChosen
&&
Workspace
.
passwordIsCorrect
(
""
))
setPassword
(
""
)
@tailrec
def
initPassword
(
implicit
preference
:
Preference
)
:
String
=
if
(
Preference
.
passwordChosen
(
preference
)
&&
Preference
.
passwordIsCorrect
(
Cypher
(
""
),
preference
))
""
else
if
(
Preference
.
passwordChosen
(
preference
))
{
val
password
=
passwordReader
.
readLine
(
"Enter your OpenMOLE password (for preferences encryption): "
,
'*'
)
val
cypher
=
Cypher
(
password
)
if
(!
Preference
.
passwordIsCorrect
(
cypher
,
preference
))
initPassword
(
preference
)
else
password
}
else
{
val
password
=
if
(
Workspace
.
passwordChosen
)
passwordReader
.
readLine
(
"Enter your OpenMOLE password (for preferences encryption): "
,
'*'
)
else
{
println
(
"OpenMOLE Password has not been set yet, choose a password."
)
askPassword
(
"Preferences password"
)
}
if
(!
setPassword
(
password
))
initPassword
println
(
"OpenMOLE Password has not been set yet, choose a password."
)
val
password
=
askPassword
(
"Preferences password"
)
val
cypher
=
Cypher
(
password
)
Preference
.
setPasswordTest
(
preference
,
cypher
)
password
}
}
object
ExitCodes
{
def
ok
=
0
...
...
@@ -98,7 +100,7 @@ object Console extends Logger {
import
org.openmole.console.Console._
class
Console
(
password
:
Option
[
String
]
=
None
,
script
:
Option
[
String
]
=
None
)
{
class
Console
(
script
:
Option
[
String
]
=
None
)
{
console
⇒
def
workspace
=
"workspace"
...
...
@@ -108,64 +110,52 @@ class Console(password: Option[String] = None, script: Option[String] = None) {
def
commandsName
=
"_commands_"
def
pluginsName
=
"_plugins_"
def
run
(
args
:
ConsoleVariables
,
workDirectory
:
Option
[
File
])
:
Int
=
{
val
correctPassword
=
password
match
{
case
None
⇒
initPassword
;
true
case
Some
(
p
)
⇒
setPassword
(
p
)
}
correctPassword
match
{
case
false
⇒
ExitCodes
.
incorrectPassword
case
true
⇒
script
match
{
case
None
⇒
val
newArgs
=
workDirectory
.
map
(
f
⇒
args
.
copy
(
workDirectory
=
f
)).
getOrElse
(
args
)
withREPL
(
newArgs
)
{
loop
⇒
loop
.
storeErrors
=
false
loop
.
loopWithExitCode
}
case
Some
(
script
)
⇒
ScalaREPL
.
warmup
val
scriptFile
=
new
File
(
script
)
val
project
=
new
Project
(
workDirectory
.
getOrElse
(
scriptFile
.
getParentFileSafe
))
project
.
compile
(
scriptFile
,
args
.
args
)
match
{
case
ScriptFileDoesNotExists
()
⇒
println
(
"File "
+
scriptFile
+
" doesn't exist."
)
ExitCodes
.
scriptDoesNotExist
case
e
:
CompilationError
⇒
println
(
e.error.stackString
)
ExitCodes.compilationError
case
compiled
:
Compiled
⇒
Try
(
compiled.eval
)
match
{
case
Success
(
res
)
⇒
val
ex
=
res.toExecution
()
Try
(
ex.start
)
match
{
def
run
(
args
:
Seq
[
String
],
workDirectory
:
Option
[
File
])(
implicit
services
:
Services
)
:
Int
=
{
import
services._
script
match
{
case
None
⇒
val
variables
=
ConsoleVariables
(
args
=
args
,
workDirectory
=
workDirectory
.
getOrElse
(
currentDirectory
))
withREPL
(
variables
)
{
loop
⇒
loop
.
storeErrors
=
false
loop
.
loopWithExitCode
}
case
Some
(
script
)
⇒
ScalaREPL
.
warmup
val
scriptFile
=
new
File
(
script
)
val
project
=
new
Project
(
workDirectory
.
getOrElse
(
scriptFile
.
getParentFileSafe
))
project
.
compile
(
scriptFile
,
args
)
match
{
case
ScriptFileDoesNotExists
()
⇒
println
(
"File "
+
scriptFile
+
" doesn't exist."
)
ExitCodes
.
scriptDoesNotExist
case
e
:
CompilationError
⇒
println
(
e.error.stackString
)
ExitCodes.compilationError
case
compiled
:
Compiled
⇒
Try
(
compiled.eval
)
match
{
case
Success
(
res
)
⇒
val
ex
=
res.toExecution
()
Try
(
ex.start
)
match
{
case
Failure
(
e
)
⇒
println
(
e.stackString
)
ExitCodes.validationError
case
Success
(
_
)
⇒
Try
(
ex.waitUntilEnded
)
match
{
case
Success
(
_
)
⇒
ExitCodes.ok
case
Failure
(
e
)
⇒
println
(
e.stackString
)
ExitCodes.validationError
case
Success
(
_
)
⇒
Try
(
ex.waitUntilEnded
)
match
{
case
Success
(
_
)
⇒
ExitCodes.ok
case
Failure
(
e
)
⇒
println
(
"
Error
during
script
execution:
"
)
print
(
e.stackString
)
ExitCodes.executionError
}
println
(
"
Error
during
script
execution:
"
)
print
(
e.stackString
)
ExitCodes.executionError
}
case
Failure
(
e
)
⇒
println
(
s
"Error during script evaluation: "
)
print
(
e
.
stackString
)
ExitCodes
.
compilationError
}
case
Failure
(
e
)
⇒
println
(
s
"Error during script evaluation: "
)
print
(
e
.
stackString
)
ExitCodes
.
compilationError
}
}
}
}
}
def
withREPL
[
T
](
args
:
ConsoleVariables
)(
f
:
ScalaREPL
⇒
T
)
=
{
...
...
openmole/bin/org.openmole.daemon/src/main/scala/org/openmole/daemon/Activator.scala
View file @
d18713b2
package
org.openmole.daemon
import
org.openmole.core.
workspa
ce.ConfigurationInfo
import
org.openmole.core.
preferen
ce.ConfigurationInfo
import
org.osgi.framework.
{
BundleActivator
,
BundleContext
}
class
Activator
extends
BundleActivator
{
...
...
openmole/bin/org.openmole.daemon/src/main/scala/org/openmole/daemon/Daemon.scala
View file @
d18713b2
...
...
@@ -17,8 +17,15 @@
package
org.openmole.daemon
import
org.openmole.core.fileservice.FileService
import
org.openmole.core.logging._
import
org.openmole.core.preference.Preference
import
org.openmole.core.serializer.SerializerService
import
org.openmole.core.workspace.
{
NewFile
,
Workspace
}
import
org.openmole.tool.file._
import
org.openmole.tool.logger._
import
org.openmole.core.services._
import
org.openmole.core.threadprovider.ThreadProvider
import
scopt._
object
Daemon
extends
Logger
{
...
...
@@ -30,6 +37,7 @@ object Daemon extends Logger {
case
class
Config
(
host
:
Option
[
String
]
=
None
,
password
:
Option
[
String
]
=
None
,
workspace
:
Option
[
File
]
=
None
,
workers
:
Int
=
1
,
cacheSize
:
Int
=
2000
)
...
...
@@ -42,6 +50,9 @@ object Daemon extends Logger {
opt
[
String
](
'p'
,
"password"
)
text
(
"password"
)
action
{
(
v
,
c
)
⇒
c
.
copy
(
password
=
Some
(
v
))
}
opt
[
String
](
'w'
,
"workspace"
)
text
(
"workspace"
)
action
{
(
v
,
c
)
⇒
c
.
copy
(
workspace
=
Some
(
File
(
v
)))
}
opt
[
Int
](
'w'
,
"workers"
)
text
(
"Number of workers, default is 1"
)
action
{
(
v
,
c
)
⇒
c
.
copy
(
workers
=
v
)
}
...
...
@@ -53,11 +64,25 @@ object Daemon extends Logger {
val
debug
=
args
.
contains
(
"-d"
)
val
filteredArgs
=
args
.
filterNot
((
_:
String
)
==
"-d"
)
parser
.
parse
(
filteredArgs
,
Config
())
foreach
{
config
⇒
new
JobLauncher
(
config
.
cacheSize
*
1024
*
1024
,
debug
).
launch
(
config
.
host
.
getOrElse
(
throw
new
RuntimeException
(
"Host undefined"
)),
config
.
password
.
getOrElse
(
throw
new
RuntimeException
(
"Password undefined"
)),
config
.
workers
)
val
workspace
=
Workspace
(
config
.
workspace
.
getOrElse
(
org
.
openmole
.
core
.
db
.
defaultOpenMOLEDirectory
))
implicit
val
preference
=
Services
.
preference
(
workspace
)
implicit
val
serializerService
=
SerializerService
()
implicit
val
newFile
=
NewFile
(
workspace
)
implicit
val
threadProvider
=
ThreadProvider
(
10
)
implicit
val
fileService
=
FileService
()
try
{
val
lancher
=
new
JobLauncher
(
config
.
cacheSize
*
1024
*
1024
,
debug
)
lancher
.
launch
(
config
.
host
.
getOrElse
(
throw
new
RuntimeException
(
"Host undefined"
)),
config
.
password
.
getOrElse
(
throw
new
RuntimeException
(
"Password undefined"
)),
config
.
workers
)
}
finally
{
workspace
.
tmpDir
.
recursiveDelete
threadProvider
.
stop
()
}
}
}
catch
{
...
...
openmole/bin/org.openmole.daemon/src/main/scala/org/openmole/daemon/JobLauncher.scala
View file @
d18713b2
...
...
@@ -28,7 +28,7 @@ import org.openmole.tool.file._
import
org.openmole.tool.logger.Logger
import
org.openmole.tool.thread._
import
org.openmole.core.tools.service.
{
OS
,
ProcessUtil
}
import
org.openmole.core.workspace.
{
ConfigurationLocation
,
Workspace
}
import
org.openmole.core.workspace.
{
NewFile
,
Workspace
}
import
org.openmole.plugin.environment.desktopgrid._
import
DesktopGridEnvironment._
import
org.openmole.plugin.environment.batch.storage._
...
...
@@ -39,6 +39,9 @@ import fr.iscpif.gridscale.ssh._
import
org.openmole.core.communication.storage._
import
org.openmole.core.communication.message._
import
org.openmole.core.communication.message.FileMessage._
import
org.openmole.core.fileservice.FileService
import
org.openmole.core.preference.
{
ConfigurationLocation
,
Preference
}
import
org.openmole.core.threadprovider.ThreadProvider
import
squants.information.Information
import
squants.time.TimeConversions._
...
...
@@ -50,7 +53,7 @@ object JobLauncher extends Logger {
val
connectionTimeout
=
ConfigurationLocation
(
"JobLauncher"
,
"connectionTimeout"
,
Some
(
1
minutes
))
}
class
JobLauncher
(
cacheSize
:
Long
,
debug
:
Boolean
)
{
class
JobLauncher
(
cacheSize
:
Long
,
debug
:
Boolean
)
(
implicit
preference
:
Preference
,
serializerService
:
SerializerService
,
newFile
:
NewFile
,
threadProvider
:
ThreadProvider
,
fileService
:
FileService
)
{
import
JobLauncher._
import
Log._
...
...
@@ -58,7 +61,8 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
val
limit
=
cacheSize
}
val
resultUploader
=
Executors
.
newSingleThreadScheduledExecutor
(
daemonThreadFactory
)
val
resultUploader
=
Executors
.
newSingleThreadScheduledExecutor
(
threadProvider
.
threadFactory
)
import
threadProvider.pool
def
launch
(
userHostPort
:
String
,
password
:
String
,
nbWorkers
:
Int
)
=
{
val
host
=
userHostPort
.
split
(
"@"
).
last
...
...
@@ -74,17 +78,19 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
val
host
=
_host
override
val
port
=
_port
def
credential
=
UserPassword
(
""
,
password
)
def
timeout
=
Workspace
.
preference
(
connectionTimeout
)
def
timeout
=
preference
(
connectionTimeout
)
}
val
root
=
""
}
val
storageFile
=
Workspac
e
.
newFile
()
S
eriali
s
erService
.
serialiseAndArchiveFiles
(
new
LocalSimpleStorage
,
storageFile
)
val
storageFile
=
newFil
e
.
newFile
()
s
eriali
z
erService
.
serialiseAndArchiveFiles
(
new
LocalSimpleStorage
,
storageFile
)
(
0
until
nbWorkers
).
foreach
{
i
⇒
background
{
runJobs
(
storageFile
,
storage
)
}
i
⇒
implicit
val
workerPool
=
Executors
.
newFixedThreadPool
(
nbWorkers
,
threadProvider
.
threadFactory
)
background
{
runJobs
(
storageFile
,
storage
)
}
}
Thread
.
currentThread
.
join
...
...
@@ -105,19 +111,19 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
val
localResultFile
=
try
{
val
localResultFile
=
Workspac
e
.
newFile
(
"job"
,
".res"
)
val
workspaceDir
=
Workspac
e
.
newDir
(
"workspace"
)
val
localResultFile
=
newFil
e
.
newFile
(
"job"
,
".res"
)
val
workspaceDir
=
newFil
e
.
newDir
(
"workspace"
)
val
osgiDir
=
new
File
(
runtime
,
UUID
.
randomUUID
.
toString
)
def
quote
(
s
:
String
)
=
if
(
OS
.
actualOS
.
isWindows
)
'"'
+
s
+
'"'
else
s
.
replace
(
" "
,
"\\ "
)
val
ptrFlag
=
if
(
OS
.
actualOS
.
is64Bit
)
"-XX:+UseCompressedOops"
else
""
val
launcher
=
s
"""-cp ${quote("launcher/*")} org.openmole.launcher.Launcher --plugins ${quote("plugins/")} --run org.openmole.runtime.SimExplorer --osgi-directory ${quote(osgiDir.getAbsolutePath)}"""
val
cmd
=
s
"java -Xmx${memory.toMegabytes.toInt}m -Dosgi.locking=none -XX:+UseG1GC $ptrFlag -Dosgi.configuration.area=${osgiDir.getName} -Dosgi.classloader.singleThreadLoads=true ${launcher} -- -s ${quote(storageFile.getAbsolutePath)} -i ${quote(localExecutionMessage.getAbsolutePath)} -o ${quote(localResultFile.getAbsolutePath)} -c ${localCommunicationDirPath} -p ${quote(pluginDir.getAbsolutePath)}"
+
(
if
(
debug
)
" -d "
else
""
)
val
cmd
=
s
"java -Xmx${memory.toMegabytes.toInt}m -Dosgi.locking=none -XX:+UseG1GC $ptrFlag -Dosgi.configuration.area=${osgiDir.getName} -Dosgi.classloader.singleThreadLoads=true ${launcher} --
--workspace ${workspaceDir.getAbsolutePath}
-s ${quote(storageFile.getAbsolutePath)} -i ${quote(localExecutionMessage.getAbsolutePath)} -o ${quote(localResultFile.getAbsolutePath)} -c ${localCommunicationDirPath} -p ${quote(pluginDir.getAbsolutePath)}"
+
(
if
(
debug
)
" -d "
else
""
)
logger
.
info
(
"Executing runtime: "
+
cmd
+
" in directory "
+
runtime
)
//val commandLine = CommandLine.parse(cmd)
val
process
=
Runtime
.
getRuntime
.
exec
(
cmd
,
Array
(
"OPENMOLE_HOME="
+
workspaceDir
.
getAbsolutePath
)
,
runtime
)
//commandLine.toString, null, runtimeLocation)
val
process
=
Runtime
.
getRuntime
.
exec
(
cmd
,
Array
.
empty
[
String
]
,
runtime
)
//commandLine.toString, null, runtimeLocation)
executeProcess
(
process
,
System
.
out
,
System
.
err
)
logger
.
info
(
"Process finished."
)
...
...
@@ -146,14 +152,14 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
next
case
None
⇒
logger
.
info
(
"Job list is empty on the remote host."
)
Thread
.
sleep
(
Workspace
.
preference
(
jobCheckInterval
).
toMillis
)
Thread
.
sleep
(
preference
(
jobCheckInterval
).
toMillis
)
background
{
fetchAJob
(
id
,
storage
)
}
}
}
catch
{
case
e
:
Exception
⇒
logger.log
(
WARNING
,
s
"
Error
while
looking
for
jobs
,
it
might
happen
if
the
jobs
have
not
yep
been
made
on
the
server
side
.
Automatic
retry
in
$
{
Workspace
.
preference
(
jobCheckInterval
)}.
", e)
Thread.sleep(
Workspace.
preference(jobCheckInterval).toMillis)
logger.log
(
WARNING
,
s
"
Error
while
looking
for
jobs
,
it
might
happen
if
the
jobs
have
not
yep
been
made
on
the
server
side
.
Automatic
retry
in
$
{
preference
(
jobCheckInterval
)}.
", e)
Thread.sleep(preference(jobCheckInterval).toMillis)
background { fetchAJob(id, storage) }
}
processJob(next)
...
...
@@ -161,7 +167,7 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
}
def uploadResult(localResultFile: File, communicationDir: String, job: String, storage: SimpleStorage) = {
val runtimeResult =
S
eriali
s
erService.deserialiseAndExtractFiles[RuntimeResult](localResultFile)
val runtimeResult =
s
eriali
z
erService.deserialiseAndExtractFiles[RuntimeResult](localResultFile)
logger.info(s"
Uploading
context
results
to
communication
dir
$communicationDir
")
...
...
@@ -193,9 +199,9 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
val resultToSend = runtimeResult.copy(result = uploadedResult)
// Upload the result
Workspac
e.withTmpFile { outputLocal ⇒
newFil
e.withTmpFile { outputLocal ⇒
logger.info("
Uploading
job
results
")
S
eriali
s
erService.serialiseAndArchiveFiles(resultToSend, outputLocal)
s
eriali
z
erService.serialiseAndArchiveFiles(resultToSend, outputLocal)
val tmpResultFile = storage.child(tmpResultsDirName, uniqName(job, "
.
res
"))
storage.upload(outputLocal, tmpResultFile)
val resultFile = storage.child(resultsDirName, uniqName(job, "
.
res
"))
...
...
@@ -242,10 +248,10 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
storage.create(storage.child(timeStempsDirName, job + timeStempSeparator + UUID.randomUUID))
val jobMessage =
Workspac
e.withTmpFile {
newFil
e.withTmpFile {
f ⇒
storage.download(storage.child(jobsDirName, job), f)
S
eriali
s
erService.deserialise[DesktopGridJobMessage](f)
s
eriali
z
erService.deserialise[DesktopGridJobMessage](f)
}
logger.info("
Job
execution
message
is
" + jobMessage.executionMessagePath)
...
...
@@ -257,7 +263,7 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
def fetchAJob(id: UUID, storage: SimpleStorage)(implicit rng: Random) = {
def download(fileMessage: FileMessage, raw: Boolean) = {
val file =
Workspac
e.newFile("
cache
", "
.
bin
")
val file =
newFil
e.newFile("
cache
", "
.
bin
")
storage.download(fileMessage.path, file, TransferOptions(raw = raw))
file → fileMessage.hash
}
...
...
@@ -271,7 +277,7 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
localCache.cache(
jobMessage.runtime,
msg ⇒ {
val dir =
Workspac
e.newDir()
val dir =
newFil
e.newDir()
logger.info("
Downloading
the
runtime
.
")
val (archive, hash) = download(msg, true)
logger.info("
Extracting
runtime
.
")
...
...
@@ -281,7 +287,7 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
)
cached ::= jobMessage.runtime
val pluginDir =
Workspac
e.newDir()
val pluginDir =
newFil
e.newDir()
pluginDir.mkdirs
jobMessage.runtimePlugins.foreach {
...
...
@@ -291,9 +297,9 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
plugin.copy(File.createTempFile("
plugin
", "
.
jar
", pluginDir))
}
val executionMessage =
Workspac
e.withTmpFile { executionMessageFileCache ⇒
val executionMessage =
newFil
e.withTmpFile { executionMessageFileCache ⇒
storage.download(jobMessage.executionMessagePath, executionMessageFileCache)
SerialiserService.deserialiseAndExtractFiles[
ExecutionMessage
]
(executionMessageFileCache)
ExecutionMessage
.load
(executionMessageFileCache)
}
def localCachedReplicatedFile(replicatedFile: ReplicatedFile, raw: Boolean) = {
...
...
@@ -305,12 +311,12 @@ class JobLauncher(cacheSize: Long, debug: Boolean) {
val files = executionMessage.files.map(localCachedReplicatedFile(_, raw = false))
val plugins = executionMessage.plugins.map(localCachedReplicatedFile(_, raw = true))
val localCommunicationDirPath =
Workspac
e.newDir()
val localCommunicationDirPath =
newFil
e.newDir()
localCommunicationDirPath.mkdirs
val localExecutionMessage =
Workspac
e.newFile("
executionMessage
", "
.
gz
"
)
val localExecutionMessage =
newFil
e.newFile("
executionMessage
", "
.
gz
"
)
S
eriali
s
erService
.
serialiseAndArchiveFiles
(
ExecutionMessage
(
plugins
,
files
,
executionMessage
.
jobs
,
localCommunicationDirPath
.
getAbsolutePath
,
executionMessage
.
runtimeSettings
),
localExecutionMessage
)
s
eriali
z
erService
.
serialiseAndArchiveFiles
(
ExecutionMessage
(
plugins
,
files
,
executionMessage
.
jobs
,
localCommunicationDirPath
.
getAbsolutePath
,
executionMessage
.
runtimeSettings
),
localExecutionMessage
)
Some
((
localExecutionMessage
,
localCommunicationDirPath
,
runtime
,
pluginDir
,
jobMessage
.
memory
,
executionMessage
,
job
,
cached
))
}
...
...
openmole/bin/org.openmole.dbserver/src/main/scala/org/openmole/dbserver/DBServer.scala
View file @
d18713b2
...
...
@@ -21,24 +21,32 @@ import java.util.logging.Logger
import
com.thoughtworks.xstream.XStream
import
java.io.File
import
java.io.FileInputStream
import
java.io.FileOutputStream
import
java.util.UUID
import
org.h2.tools.Server
import
org.openmole.core.replication.
{
DBServerRunning
,
replicas
,
DBServerInfo
}
import
slick.driver.H2Driver.api._
import
scala.concurrent.Await
import
scala.concurrent.duration.Duration
import
scala.util.
{
Success
,
Failure
,
Try
}
import
scala.util.
{
Failure
,
Success
,
Try
}
import
org.openmole.core.db
import
org.openmole.core.db.
{
DBServerInfo
,
DBServerRunning
}
object
DBServer
extends
App
{
val
base
=
if
(!
args
.
isEmpty
)
new
File
(
args
(
0
))
else
db
.
defaultOpenMOLEDirectory
def
checkInterval
=
30000
def
maxAllDead
=
5
val
base
=
DBServerInfo
.
dbDirectory
val
dbDirectory
=
db
.
dbDirectory
(
base
)
dbDirectory
.
mkdirs
()
val
lockFile
=
DBServerInfo
.
dbLockFile
def
dbLock
=
s
"${db.dbName}.lock"
def
urlDBPath
=
s
"${db.dbName};MV_STORE=FALSE;MVCC=TRUE;"
val
lockFile
=
new
File
(
dbDirectory
,
dbLock
)
lockFile
.
createNewFile